def initialize_diary(): diary = Diary(name='digits_vs_letters', path='results', overwrite=False, fig_format='svg') diary.add_notebook('training', verbose=True) diary.add_notebook('validation', verbose=True) return diary
def setUp(self): self.EPSILON = 0.01 sub1 = Subject("Biologia", [[3.5, 2.5, 5.0], [1, 1, 1, 1, 1, 1, 1, 1, 0]]) sub2 = Subject("Matematyka", [[2.5, 3.0, 2.5], [1, 1, 0, 1, 0, 1, 0, 1, 1]]) self.subjects = [sub1, sub2] self.andrzej = Student("Andrzej Abacki", self.subjects) self.diary = Diary("AGH", 2016, "data.json")
def __init__(self): Gtk.Window.__init__(self, title='Diary') self.set_default_size(800, 600) self.connect('destroy', Gtk.main_quit) main_box = Gtk.Box() size_group = Gtk.SizeGroup(Gtk.SizeGroupMode.HORIZONTAL) # initializing all relevant classes diary = Diary("Test User") textview = Textview() header = Header() searchbar = Searchbar() sidebox = Sidebox() # transfer classes to linker linker = Linker(diary, header, sidebox, searchbar, textview) # transfer linker to header and sidebox header.set_connection_linker(linker) sidebox.set_connection_linker(linker) sidebox.update_year() searchbar.set_connection_linker(linker) searchbar.set_revealer_signal() # connect size_group to header and sidebox header.set_size_group(size_group) sidebox.set_size_group(size_group) sidebox.set_revealer_signal() # setup relevant buttons in header header.set_backbutton() header.set_forwardbutton() header.set_searchbutton() header.set_addbutton() header.set_editbutton() self.set_titlebar(header) # create new side_box to add searchbar and sidebox # and add it to beginning of mainbox side_box = Gtk.VBox() side_box.pack_start(searchbar, False, False, 0) side_box.pack_start(sidebox, True, True, 0) side_box.set_hexpand(False) main_box.pack_start(side_box, False, False, 0) # add separator between side_box and textview separator = Gtk.HSeparator() separator.set_size_request(1, 0) main_box.pack_start(separator, False, False, 0) # add textview to end of mainbox main_box.pack_start(textview, False, True, 0) self.add(main_box)
def initialize_diary(): diary = Diary(STUDENTS, CLASSES, DATES) for i in xrange(NUMBER_OF_SCORES): diary.add_score(random.choice(CLASSES), random.choice(STUDENTS), random.choice(SCORES)) for clazz in CLASSES: for date in DATES: for student in STUDENTS: if random.choice([True, False]): diary.add_attendance(clazz, student, date) return diary
def do_it(self, ev): login = self.login_edit.get() password = self.password_edit.get() diary_id = self.diary_id_edit.get() filename = self.filename_edit.get() split_type = self.split_type.get() if not login: messagebox.showinfo("Error", "Логин не задан") return if not diary_id: messagebox.showinfo("Error", "Адрес сообщества не задан") return if not filename: messagebox.showinfo("Error", "Путь к файлу не задан") return api = Diary() try: api.login(login, password) text_with_header = util.load(filename) prefix = os.path.splitext(filename)[0] text_with_header = util.fix_characters(text_with_header) header, text = find_header(text_with_header) if split_type == 1: post, comments = split_text_with_comments(header, text) util.store(prefix + "_post.txt", post) for i, comment in enumerate(comments): util.store(prefix + "_comment_%d.txt" % (i+1), comment) # Send to diary post_id = api.new_post(post, diary_id) for comment in comments: api.add_comment(post_id, comment) if len(comments) > 0: messagebox.showinfo("Info", "Пост успешно опубликован, тексты комментариев ищите в файлах *comment_N.txt") else: messagebox.showinfo("Info", "Пост успешно опубликован") else: posts = split_text_with_posts(header, text) for i, post in enumerate(posts): util.store(prefix + "_post_%d.txt" % (i + 1), post) # Send to diary for post in posts: api.new_post(post, diary_id) messagebox.showinfo("Info", "Посты успешно опубликованы. Тексты продублированы в файлы *post_N.txt") except Exception as e: messagebox.showinfo("Error", str(e)) return
def _retrieve_from_diary(year, number): """Retrieve the data from the data indicated. """ url_base = compose_url(URL_BASE, year, number) dia = Diary(year, number, url_base) print "Retrieving contents from: %s" % url_base for section in sorted(SECTIONS.keys()): url = compose_url(url_base, section) dia.add_section(SECTIONS[section]) WScrap.scrap_page(url, dia) return dia
def main(dataset_names=None): if dataset_names is None: dataset_names = [ 'autos', 'car', 'cleveland', 'dermatology', 'ecoli', 'flare', 'glass', 'led7digit', 'lymphography', 'nursery', 'page-blocks', 'pendigits', 'satimage', 'segment', #'shuttle', 'vehicle', 'vowel', 'yeast', 'zoo', 'auslan' ] seed_num = 42 mc_iterations = 5 n_folds = 2 estimator_type = "svm" # Diary to save the partial and final results diary = Diary(name='results_Krawczyk2015', path='results', overwrite=False, fig_format='svg') # Hyperparameters for this experiment (folds, iterations, seed) diary.add_notebook('parameters', verbose=True) # Summary for each dataset diary.add_notebook('datasets', verbose=False) # Partial results for validation diary.add_notebook('validation', verbose=True) # Final results diary.add_notebook('summary', verbose=True) columns = ['dataset', 'method', 'mc', 'test_fold', 'acc'] df = MyDataFrame(columns=columns) diary.add_entry('parameters', [ 'seed', seed_num, 'mc_it', mc_iterations, 'n_folds', n_folds, 'estimator_type', estimator_type ]) data = Data(dataset_names=dataset_names) for i, (name, dataset) in enumerate(data.datasets.iteritems()): np.random.seed(seed_num) dataset.print_summary() diary.add_entry('datasets', [dataset.__str__()]) accuracies = np.zeros(mc_iterations * n_folds) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) if estimator_type == "svm": est = OneClassSVM(nu=0.5, gamma=0.5) elif estimator_type == "gmm": est = GMM(n_components=3) bc = BackgroundCheck(estimator=est) oc = OcDecomposition(base_estimator=bc) oc.fit(x_train, y_train) accuracy = oc.accuracy(x_test, y_test) accuracies[mc * n_folds + test_fold] = accuracy diary.add_entry('validation', [ 'dataset', name, 'method', 'our', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy ]) df = df.append_rows([[name, 'our', mc, test_fold, accuracy]]) df = df.convert_objects(convert_numeric=True) table = df.pivot_table(values=['acc'], index=['dataset'], columns=['method'], aggfunc=[np.mean, np.std]) diary.add_entry('summary', [table])
def main(): dataset_names = ['diabetes', 'ecoli', 'glass', 'heart-statlog', 'ionosphere', 'iris', 'letter', 'mfeat-karhunen', 'mfeat-morphological', 'mfeat-zernike', 'optdigits', 'pendigits', 'sonar', 'vehicle', 'waveform-5000'] data = Data(dataset_names=dataset_names) diary = Diary(name='hempstalk', path='results', overwrite=False, fig_format='svg') diary.add_notebook('cross_validation') # Columns for the DataFrame columns=['Dataset', 'MC iteration', 'N-fold id', 'Actual class', 'Model', 'AUC', 'Prior'] # Create a DataFrame to record all intermediate results df = pd.DataFrame(columns=columns) mc_iterations = 10 n_folds = 10 gammas = {"diabetes":0.00005, "ecoli":0.1, "glass":0.005, "heart-statlog":0.0001, "ionosphere":0.00005, "iris":0.0005, "letter":0.000005, "mfeat-karhunen":0.0001, "mfeat-morphological":0.0000001, "mfeat-zernike":0.000001, "optdigits":0.00005, "pendigits":0.000001, "sonar":0.001, "vehicle":0.00005, "waveform-5000":0.001} for i, (name, dataset) in enumerate(data.datasets.iteritems()): print('Dataset number {}'.format(i)) data.sumarize_datasets(name) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) n_training = np.alen(y_train) w_auc_fold_dens = 0 w_auc_fold_bag = 0 w_auc_fold_com = 0 prior_sum = 0 for actual_class in dataset.classes: tr_class = x_train[y_train == actual_class, :] tr_class_unique_values = [np.unique(tr_class[:,column]).shape[0] for column in range(tr_class.shape[1])] cols_keep = np.where(np.not_equal(tr_class_unique_values,1))[0] tr_class = tr_class[:,cols_keep] x_test_cleaned = x_test[:,cols_keep] t_labels = (y_test == actual_class).astype(int) prior = np.alen(tr_class) / n_training if np.alen(tr_class) > 1 and not all(t_labels == 0): prior_sum += prior n_c = tr_class.shape[1] if n_c > np.alen(tr_class): n_c = np.alen(tr_class) # Train a Density estimator model_gmm = GMM(n_components=1, covariance_type='diag') model_gmm.fit(tr_class) sv = OneClassSVM(nu=0.1, gamma=0.5) bc = BackgroundCheck(estimator=sv) bc.fit(tr_class) svm_scores = bc.predict_proba(x_test_cleaned)[:, 1] # Generate artificial data new_data = model_gmm.sample(np.alen(tr_class)) # Train a Bag of Trees bag = BaggingClassifier( base_estimator=DecisionTreeClassifier(), n_estimators=10) new_data = np.vstack((tr_class, new_data)) y = np.zeros(np.alen(new_data)) y[:np.alen(tr_class)] = 1 bag.fit(new_data, y) # Combine the results probs = bag.predict_proba(x_test_cleaned)[:, 1] scores = model_gmm.score(x_test_cleaned) com_scores = (probs / np.clip(1.0 - probs, np.float32(1e-32), 1.0)) * (scores-scores.min()) # Generate our new data # FIXME solve problem with #samples < #features pca=True if tr_class.shape[0] < tr_class.shape[1]: pca=False our_new_data = reject.create_reject_data( tr_class, proportion=1, method='uniform_hsphere', pca=pca, pca_variance=0.99, pca_components=0, hshape_cov=0, hshape_prop_in=0.99, hshape_multiplier=1.5) our_new_data = np.vstack((tr_class, our_new_data)) y = np.zeros(np.alen(our_new_data)) y[:np.alen(tr_class)] = 1 # Train Our Bag of Trees our_bag = BaggingClassifier( base_estimator=DecisionTreeClassifier(), n_estimators=10) our_bag.fit(our_new_data, y) # Combine the results our_probs = our_bag.predict_proba(x_test_cleaned)[:, 1] our_comb_scores = (our_probs / np.clip(1.0 - our_probs, np.float32(1e-32), 1.0)) * (scores-scores.min()) # Scores for the Density estimator auc_dens = roc_auc_score(t_labels, scores) # Scores for the Bag of trees auc_bag = roc_auc_score(t_labels, probs) # Scores for the Combined model auc_com = roc_auc_score(t_labels, com_scores) # Scores for our Bag of trees (trained on our data) auc_our_bag = roc_auc_score(t_labels, our_probs) # Scores for our Bag of trees (trained on our data) auc_our_comb = roc_auc_score(t_labels, our_comb_scores) # Scores for the Background Check with SVm auc_svm = roc_auc_score(t_labels, svm_scores) # Create a new DataFrame to append to the original one dfaux = pd.DataFrame([[name, mc, test_fold, actual_class, 'Combined', auc_com, prior], [name, mc, test_fold, actual_class, 'P(T$|$X)', auc_bag, prior], [name, mc, test_fold, actual_class, 'P(X$|$A)', auc_dens, prior], [name, mc, test_fold, actual_class, 'Our Bagg', auc_our_bag, prior], [name, mc, test_fold, actual_class, 'Our Combined', auc_our_comb, prior], [name, mc, test_fold, actual_class, 'SVM_BC', auc_svm, prior]], columns=columns) df = df.append(dfaux, ignore_index=True) # generate_and_save_plots(t_labels, scores, diary, name, mc, test_fold, # actual_class, 'P(X$|$A)') # generate_and_save_plots(t_labels, probs, diary, name, mc, test_fold, # actual_class, 'P(T$|$X)') # generate_and_save_plots(t_labels, com_scores, diary, name, mc, test_fold, # actual_class, 'Combined') # generate_and_save_plots(t_labels, our_probs, diary, name, mc, test_fold, # actual_class, 'Our_Bagg') # generate_and_save_plots(t_labels, our_comb_scores, diary, name, mc, test_fold, # actual_class, 'Our_Combined') # generate_and_save_plots(t_labels, svm_scores, diary, # name, mc, test_fold, # actual_class, 'SVM_BC') # Convert values to numeric df = df.convert_objects(convert_numeric=True) # Group everything except classes dfgroup_classes = df.groupby(by=['Dataset', 'MC iteration', 'N-fold id', 'Model']) # Compute the Prior sum for each dataset, iteration and fold df['Prior_sum'] = dfgroup_classes['Prior'].transform(np.sum) # Compute the individual weighted AUC per each class and experiment df['wAUC'] = df.Prior * df.AUC / df.Prior_sum # Sum the weighted AUC of each class per each experiment series_wAUC = dfgroup_classes['wAUC'].sum() # Transform the series to a DataFrame df_wAUC = series_wAUC.reset_index(inplace=False) # Compute mean and standard deviation of wAUC per Dataset and model final_results = df_wAUC.groupby(['Dataset', 'Model'])['wAUC'].agg([np.mean, np.std]) # Transform the series to a DataFrame final_results.reset_index(inplace=True) # Represent the results in a table format final_table = final_results.pivot_table(values=['mean', 'std'], index=['Dataset'], columns=['Model']) # Export the results in a csv and LaTeX file export_results(final_table)
y = np.hstack((np.ones(np.alen(x)), np.zeros(np.alen(r)))).T model_rej.fit(xr, y) return model_rej def train_classifier_model(x, y): model_clas = svm.SVC(probability=True) #model_clas = tree.DecisionTreeClassifier(max_depth=3) model_clas = model_clas.fit(x, y) return model_clas if __name__ == "__main__": diary = Diary(name='test_rgrpg', path='results', overwrite=False, fig_format='svg') diary.add_notebook('training') diary.add_notebook('validation') # for i in [6]: #range(1,4): n_iterations = 1 n_thresholds = 100 accuracies = np.empty((n_iterations, n_thresholds)) recalls = np.empty((n_iterations, n_thresholds)) for example in [2, 3, 4, 5, 6, 7, 8, 9]: np.random.seed(42) print('Runing example = {}'.format(example)) for iteration in range(n_iterations): #####################################################
# - get students average score in class # - hold students name and surname # - Count total attendance of student # The default interface for interaction should be python interpreter. # Please, use your imagination and create more functionalities. # Your project should be able to handle entire school. # If you have enough courage and time, try storing (reading/writing) # data in text files (YAML, JSON). # If you have even more courage, try implementing user interface. # #Try to expand your implementation as best as you can. #Think of as many features as you can, and try implementing them. #Make intelligent use of pythons syntactic sugar (overloading, iterators, generators, etc) #Most of all: CREATE GOOD, RELIABLE, READABLE CODE. #The goal of this task is for you to SHOW YOUR BEST python programming skills. #Impress everyone with your skills, show off with your code. # #Your program must be runnable with command "python task.py". #Show some usecases of your library in the code (print some things) # #When you are done upload this code to your github repository. # #Delete these comments before commit! #Good luck. from diary import Diary, Student, SchoolClass diary = Diary() schoolclass = SchoolClass("biology") student = Student("majlosz", "ef") #schoolclass.add_students([student])
def main(dataset_names=None, estimator_type="gmm", mc_iterations=20, n_folds=5, n_ensemble=100, seed_num=42): if dataset_names is None: # All the datasets used in Li2014 datasets_li2014 = [ 'abalone', 'balance-scale', 'credit-approval', 'dermatology', 'ecoli', 'german', 'heart-statlog', 'hepatitis', 'horse', 'ionosphere', 'lung-cancer', 'libras-movement', 'mushroom', 'diabetes', 'landsat-satellite', 'segment', 'spambase', 'wdbc', 'wpbc', 'yeast' ] datasets_hempstalk2008 = [ 'diabetes', 'ecoli', 'glass', 'heart-statlog', 'ionosphere', 'iris', 'letter', 'mfeat-karhunen', 'mfeat-morphological', 'mfeat-zernike', 'optdigits', 'pendigits', 'sonar', 'vehicle', 'waveform-5000' ] datasets_others = [ 'diabetes', 'ecoli', 'glass', 'heart-statlog', 'ionosphere', 'iris', 'letter', 'mfeat-karhunen', 'mfeat-morphological', 'mfeat-zernike', 'optdigits', 'pendigits', 'sonar', 'vehicle', 'waveform-5000', 'scene-classification', 'tic-tac', 'autos', 'car', 'cleveland', 'dermatology', 'flare', 'page-blocks', 'segment', 'shuttle', 'vowel', 'zoo', 'abalone', 'balance-scale', 'credit-approval', 'german', 'hepatitis', 'lung-cancer' ] # Datasets that we can add but need to be reduced datasets_to_add = ['MNIST'] dataset_names = list( set(datasets_li2014 + datasets_hempstalk2008 + datasets_others)) # Diary to save the partial and final results diary = Diary(name='results_Li2014', path='results', overwrite=False, fig_format='svg') # Hyperparameters for this experiment (folds, iterations, seed) diary.add_notebook('parameters', verbose=True) # Summary for each dataset diary.add_notebook('datasets', verbose=False) # Partial results for validation diary.add_notebook('validation', verbose=True) # Final results diary.add_notebook('summary', verbose=True) columns = ['dataset', 'method', 'mc', 'test_fold', 'acc', 'logloss'] df = MyDataFrame(columns=columns) diary.add_entry('parameters', [ 'seed', seed_num, 'mc_it', mc_iterations, 'n_folds', n_folds, 'n_ensemble', n_ensemble, 'estimator_type', estimator_type ]) data = Data(dataset_names=dataset_names) for name, dataset in data.datasets.iteritems(): if name in ['letter', 'shuttle']: dataset.reduce_number_instances(0.1) export_datasets_description_to_latex(data, path=diary.path) for i, (name, dataset) in enumerate(data.datasets.iteritems()): np.random.seed(seed_num) dataset.print_summary() diary.add_entry('datasets', [dataset.__str__()]) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) # Binary discriminative classifier sv = SVC(kernel='linear', probability=True) # Density estimator for the background check if estimator_type == "svm": gamma = 1.0 / x_train.shape[1] est = OneClassSVM(nu=0.1, gamma=gamma) elif estimator_type == "gmm": est = GMM(n_components=1) elif estimator_type == "gmm3": est = GMM(n_components=3) elif estimator_type == "mymvn": est = MyMultivariateNormal() # Multiclass discriminative model with one-vs-one binary class. ovo = OvoClassifier(base_classifier=sv) classifier = ConfidentClassifier(classifier=ovo, estimator=est, mu=0.5, m=0.5) ensemble = Ensemble(base_classifier=classifier, n_ensemble=n_ensemble) # classifier = ConfidentClassifier(classifier=sv, # estimator=est, mu=0.5, # m=0.5) # ovo = OvoClassifier(base_classifier=classifier) # ensemble = Ensemble(base_classifier=ovo, # n_ensemble=n_ensemble) xs_bootstrap, ys_bootstrap = ensemble.fit(x_train, y_train) accuracy = ensemble.accuracy(x_test, y_test) log_loss = ensemble.log_loss(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'our', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy, 'logloss', log_loss ]) df = df.append_rows( [[name, 'our', mc, test_fold, accuracy, log_loss]]) # Li2014: EP-CC model # The classification confidence is used in learning the weights # of the base classifier as well as in weighted voting. ensemble_li = Ensemble(n_ensemble=n_ensemble, lambd=1e-8) ensemble_li.fit(x_train, y_train, xs=xs_bootstrap, ys=ys_bootstrap) accuracy_li = ensemble_li.accuracy(x_test, y_test) log_loss_li = ensemble_li.log_loss(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'Li2014', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy_li, 'logloss', log_loss_li ]) df = df.append_rows( [[name, 'Li2014', mc, test_fold, accuracy_li, log_loss_li]]) export_summary(df, diary)
def main(dataset_names=None, estimator_type="kernel", mc_iterations=1, n_folds=10, seed_num=42): if dataset_names is None: dataset_names = ['glass', 'hepatitis', 'ionosphere', 'vowel'] bandwidths_o_norm = { 'glass': 0.09, 'hepatitis': 0.105, 'ionosphere': 0.039, 'vowel': 0.075 } bandwidths_bc = { 'glass': 0.09, 'hepatitis': 0.105, 'ionosphere': 0.039, 'vowel': 0.0145 } bandwidths_t_norm = { 'glass': 0.336, 'hepatitis': 0.015, 'ionosphere': 0.0385, 'vowel': 0.0145 } tuned_mus = { 'glass': [0.094, 0.095, 0.2, 0.0, 0.0, 0.1], 'vowel': [0.0, 0.0, 0.5, 0.5, 0.5, 0.0] } tuned_ms = { 'glass': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'vowel': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0] } bandwidth_o_norm = 0.05 bandwidth_t_norm = 0.05 bandwidth_bc = 0.05 # Diary to save the partial and final results diary = Diary(name='results_Tax2008', path='results', overwrite=False, fig_format='svg') # Hyperparameters for this experiment (folds, iterations, seed) diary.add_notebook('parameters', verbose=True) # Summary for each dataset diary.add_notebook('datasets', verbose=False) # Partial results for validation diary.add_notebook('validation', verbose=True) # Final results diary.add_notebook('summary', verbose=True) columns = ['dataset', 'method', 'mc', 'test_fold', 'acc'] df = MyDataFrame(columns=columns) diary.add_entry('parameters', [ 'seed', seed_num, 'mc_it', mc_iterations, 'n_folds', n_folds, 'estimator_type', estimator_type, 'bw_o', bandwidth_o_norm, 'bw_t', bandwidth_t_norm, 'bw_bc', bandwidth_bc ]) data = Data(dataset_names=dataset_names) for name, dataset in data.datasets.iteritems(): if name in ['letter', 'shuttle']: dataset.reduce_number_instances(0.1) export_datasets_description_to_latex(data, path=diary.path) for i, (name, dataset) in enumerate(data.datasets.iteritems()): np.random.seed(seed_num) dataset.print_summary() diary.add_entry('datasets', [dataset.__str__()]) # accuracies_tuned = np.zeros(mc_iterations * n_folds) # if name in bandwidths_o_norm.keys(): # bandwidth_o_norm = bandwidths_o_norm[name] # bandwidth_t_norm = bandwidths_t_norm[name] # bandwidth_bc = bandwidths_bc[name] # else: # bandwidth_o_norm = np.mean(bandwidths_o_norm.values()) # bandwidth_t_norm = np.mean(bandwidths_t_norm.values()) # bandwidth_bc = np.mean(bandwidths_bc.values()) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) # if name in ['glass', 'hepatitis', 'ionosphere', 'thyroid', # 'iris', 'heart-statlog', 'diabetes', 'abalone', # 'mushroom', 'spambase']: x_test, y_test = generate_outliers(x_test, y_test) # elif name == 'vowel': # x_train = x_train[y_train <= 5] # y_train = y_train[y_train <= 5] # y_test[y_test > 5] = 6 # elif dataset.n_classes > 2: # x_train = x_train[y_train <= dataset.n_classes/2] # y_train = y_train[y_train <= dataset.n_classes/2] # y_test[y_test > dataset.n_classes/2] = dataset.n_classes+1 # else: # continue if estimator_type == "svm": est = OneClassSVM(nu=0.5, gamma=1.0 / x_train.shape[1]) elif estimator_type == "gmm": est = GMM(n_components=1) elif estimator_type == "gmm3": est = GMM(n_components=3) elif estimator_type == "kernel": est = MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_bc) estimators = None bcs = None if estimator_type == "kernel": estimators, bcs = fit_estimators( MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_bc), x_train, y_train) # Untuned background check bc = BackgroundCheck(estimator=est, mu=0.0, m=1.0) oc = OcDecomposition(base_estimator=bc) if estimators is None: oc.fit(x_train, y_train) else: oc.set_estimators(bcs, x_train, y_train) accuracy = oc.accuracy(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'BC', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy ]) df = df.append_rows([[name, 'BC', mc, test_fold, accuracy]]) e = MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_o_norm) oc_o_norm = OcDecomposition(base_estimator=e, normalization="O-norm") if estimators is None: oc_o_norm.fit(x_train, y_train) else: oc_o_norm.set_estimators(estimators, x_train, y_train) accuracy_o_norm = oc_o_norm.accuracy(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'O-norm', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy_o_norm ]) df = df.append_rows( [[name, 'O-norm', mc, test_fold, accuracy_o_norm]]) e = MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_t_norm) oc_t_norm = OcDecomposition(base_estimator=e, normalization="T-norm") if estimators is None: oc_t_norm.fit(x_train, y_train) else: oc_t_norm.set_estimators(estimators, x_train, y_train) accuracy_t_norm = oc_t_norm.accuracy(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'T-norm', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy_t_norm ]) df = df.append_rows( [[name, 'T-norm', mc, test_fold, accuracy_t_norm]]) # Tuned background check # if name in tuned_mus.keys(): # mus = tuned_mus[name] # ms = tuned_ms[name] # else: # mus = None # ms = None # bc = BackgroundCheck(estimator=est, mu=0.0, m=1.0) # oc_tuned = OcDecomposition(base_estimator=bc) # oc_tuned.fit(x_train, y_train, mus=mus, ms=ms) # accuracy_tuned = oc_tuned.accuracy(x_test, y_test, mus=mus, # ms=ms) # accuracies_tuned[mc * n_folds + test_fold] = accuracy_tuned # diary.add_entry('validation', ['dataset', name, # 'method', 'BC-tuned', # 'mc', mc, # 'test_fold', test_fold, # 'acc', accuracy_tuned]) # df = df.append_rows([[name, 'BC-tuned', mc, test_fold, # accuracy_tuned]]) export_summary(df, diary)
try: perms = [ perm['name'] for perm in vk.method('groups.getTokenPermissions')['permissions'] ] if 'manage' not in perms or 'messages' not in perms: call_exit('У ключа недостаточно прав') except ApiError: call_exit('Неверный ключ доступа') try: vk.method('groups.getOnlineStatus', {'group_id': parser['Vk']['group_id']}) except Exception: call_exit('В настройках группы отключены сообщения или неверный id группы') d = Diary(parser['Diary']['diary_login'], parser['Diary']['diary_password'], session) try: d.auth() except ValueError: call_exit('Неверный логин или пароль') except requests.exceptions.HTTPError: call_exit('Электронный дневник не работает. Попробуйте запустить позже') payload = { 'group_id': parser['Vk']['group_id'], 'enabled': 1, 'api_version': '5.92', 'message_new': 1 } try: vk.method('groups.setLongPollSettings', payload)