def __init__(self, seed=999, n_splits=10): self.seed = seed self.n_splits = n_splits import sys sys.path.append( r"C:\Users\Kelvin\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation" ) #sys.path.append(r"C:\Users\Kelvi\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation") from Models import Models self.models = Models() from sklearn.model_selection import GridSearchCV from sklearn.model_selection import TimeSeriesSplit tscv = TimeSeriesSplit(n_splits) from sklearn.neural_network import MLPClassifier from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF from sklearn.tree import DecisionTreeClassifier from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier from sklearn.naive_bayes import GaussianNB from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis #self.models.add_model(model = GridSearchCV(estimator=MLPClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Neural Net') #self.models.add_model(model = GridSearchCV(estimator=KNeighborsClassifier(), param_grid={}, cv=tscv), model_name = 'KNN') #self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='linear', random_state=seed), param_grid={}, cv=tscv), model_name = 'Linear SVM') #self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='rbf', random_state=seed), param_grid={}, cv=tscv), model_name = 'RBF SVM') #self.models.add_model(model = GridSearchCV(estimator=GaussianProcessClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Gaussian Process') #self.models.add_model(model = GridSearchCV(estimator=DecisionTreeClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Decision Tree') #self.models.add_model(model = GridSearchCV(estimator=RandomForestClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Random Forest') #self.models.add_model(model = GridSearchCV(estimator=AdaBoostClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'AdaBoost') #self.models.add_model(model = GridSearchCV(estimator=GaussianNB(), param_grid={}, cv=tscv), model_name = 'Naive Bayes') ##self.models.add_model(model = GridSearchCV(estimator=QuadraticDiscriminantAnalysis(), param_grid={}, cv=tscv), model_name = 'QDA') self.models.add_model(model=MLPClassifier(random_state=seed), model_name='Neural Net') self.models.add_model(model=KNeighborsClassifier(), model_name='KNN') self.models.add_model(model=SVC(kernel='linear', random_state=seed), model_name='Linear SVM') self.models.add_model(model=SVC(kernel='rbf', random_state=seed), model_name='RBF SVM') self.models.add_model( model=GaussianProcessClassifier(random_state=seed), model_name='Gaussian Process') self.models.add_model(model=DecisionTreeClassifier(random_state=seed), model_name='Decision Tree') self.models.add_model(model=RandomForestClassifier(random_state=seed), model_name='Random Forest') self.models.add_model(model=AdaBoostClassifier(random_state=seed), model_name='AdaBoost') self.models.add_model(model=GaussianNB(), model_name='Naive Bayes')
def train_model(new_data): #declare objects Data_preparation = data_preparation() models = Models() if new_data: #read_data data = Data_preparation.read_data_add_labels() add_article_topic_col(data) data = Data_preparation.add_full_text(data) data = Data_preparation.add_binary_topics_col(data) data.to_csv('new_data/new_processed_data.csv') else: data = pd.read_csv('new_data/new_processed_data.csv', index_col=0) #for fast debug #data = data.sample(n=1000) train, test = train_test_split(data, test_size=0.1) train1, train2 = train_test_split(train, test_size=0.5) #train naive baise model nb_model_obj = models.train_NB_model(train1) zero_one_train_matrix = Data_preparation.create_zero_one_matrix( nb_model_obj, train2) lr_model_obj = models.train_lr_model(zero_one_train_matrix, train2['LABEL']) #save model if save_model: nb_pkl_filename = 'nb_pickle_model.pkl' with open(nb_pkl_filename, 'wb') as file: pickle.dump(nb_model_obj, file) lr_pkl_filename = 'lr_pickle_model.pkl' with open(lr_pkl_filename, 'wb') as file: pickle.dump(lr_model_obj, file) predict_obj = Predict(nb_model_obj, lr_model_obj) nb_prediction = predict_obj.nb_predict(test, data_preparation) print('test nb score: ' + str(np.mean(nb_prediction == test['LABEL']))) lr_proba, lr_prediction = predict_obj.lr_predict(test, Data_preparation) print('test lr score: ' + str(np.mean(lr_prediction == test['LABEL']))) predict_obj.get_confusion_matrix(test['LABEL'], lr_prediction, 'all') quantile_data, quantile_accurate = predict_obj.get_quantile_accurate( test, lr_prediction, lr_proba) with pd.option_context('display.max_rows', None, 'display.max_columns', None): print(quantile_accurate) #todo add confusion matrix for each band for index, row in quantile_accurate.iterrows(): print(row['probaBand']) quantile = quantile_data[quantile_data['probaBand'] == row['probaBand']]
def test_challenge5(self): self.driver.get("https://www.copart.com") s = CopartSearchBar(self.driver) model = "porsche" s.search_input(model) entryNumber100 = self.driver.find_element( By.XPATH, "//*[@id='serverSideDataTable_length']//option[3]") entryNumber100.click() WebDriverWait(self.driver, 10).until( expected_conditions.visibility_of_element_located( (By.XPATH, "//*[@id='serverSideDataTable']/tbody/tr[100]"))) m = Models(self.driver) m.unique_model_counter(100) d = Damages(self.driver) d.damage_finder(100)
def __init__(self, formal, caps, group): # Load the dataset data = DatasetLoader() # Create and train the models modelz = Models(data) # modelz.showPerformances() modelz.setSingleModel() # this will set the multiNB model # Wait for the models to finish loading while not modelz.endLoading: time.sleep(1) # Initialize the chat and run the dialogs chat = ChatManager(modelz, group, formal, caps) chat.run()
def Classify(X, Y, cls, rep, k=5000): # Start moment Start_moment = time.time() title = 'Classificando com {} e {} k={}'.format(cls, rep, k) print(title) # Creating the K-fold cross validator if 'w2v' in rep: train_x = load(open('w2v_rep/{}_train_x.pkl'.format(rep), 'rb')) train_y = load(open('w2v_rep/{}_train_y.pkl'.format(rep), 'rb')) test_x = load(open('w2v_rep/{}_test_x.pkl'.format(rep), 'rb')) test_y = load(open('w2v_rep/{}_test_y.pkl'.format(rep), 'rb')) else: X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=123, stratify=Y) train_x, train_y, test_x, test_y = Representations( ).get_representation(rep=rep, train_x=X_train, train_y=y_train, test_x=X_test, test_y=y_test, k=k, cat=None) sm = SMOTE(sampling_strategy='minority', random_state=None) train_x, train_y = sm.fit_sample(train_x, train_y) # dump(train_x, open('w2v_rep/{}_train_x.pkl'.format(rep), 'wb')) # dump(train_y, open('w2v_rep/{}_train_y.pkl'.format(rep), 'wb')) # dump(test_x, open('w2v_rep/{}_test_x.pkl'.format(rep), 'wb')) # dump(test_y, open('w2v_rep/{}_test_y.pkl'.format(rep), 'wb')) # return classifier = Models().get_classifier(cls) classifier.fit(train_x, train_y) # Train_Classifier(classifier, X_train, Y_train) pred = classifier.predict(test_x) # report = classification_report(test_labels, test_pred, target_names=['Contrário', 'Favorável'] if plb =='polaridade' else ['neutro', 'opiniao']) report = classification_report(test_y, pred, target_names=['no', 'yes']) print(report) Finish_moment = time.time() tm = "It took " + str((Finish_moment - Start_moment)) + " seconds" print(tm)
def event_details(): name = [ "Ed-Sheeran Concert", "MAGA Rally", "Chug-the-Jug", "Cupid's Ball", "Deep Blue Destiny", "TGIF-Midnight", "Mexican-Fiesta", "Paradise Awaits", "Frosty Frozen Prom", "Tequila Sunrise", "Light Up the Night", "Zombie Prom" ] date = [ "5-Oct-2018", "9-Oct-2018", "26-Oct-2018", "8-Nov-2018", "18-Nov-2018", "4-Nov-2018", "21-Nov-2018", "26-Dec-2018", "8-Jan-2019" ] price = [80, 100, 60, 150, 30, 45, 90, 200] location = [ "Nashville, TN", "Memphis, TN", "Huntsville, AL", "New York, NY", "Salt Lake City, UT", "Denver, CO" ] events = [] for i in range(0, len(name)): events.append( M.Event(name[i], date[random.randint(0, len(date) - 1)], location[random.randint(0, len(location) - 1)], price[random.randint(0, len(price) - 1)])) return events
def __init__(self, modelType=None, numClass=None, numCell=None, maxLen=None, test=False): if not test: self.model = Models.getModel(modelType, numClass, numCell, maxLen).cuda() self.criterion = nn.CrossEntropyLoss() self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0001) else: self.model = None
def retrieveAccount(username): sql = """ select * from Account where username = '******' """ % username record = pd.read_sql(sql, engine) returnValue = M.Account(record['FName'][0], record['LName'][0], record['email'][0], record['phone'][0], record['UserName'][0], record['pword'][0], record['OAuth'][0]) return returnValue
def post(self): # adicionando dívidas simples args = parser.parse_args() DebtsValues, uniqueId = Models.AddDebtsValuesModels(args) AddValues = SendDbServices.SendSimpleDebts(connection, DebtsValues[uniqueId]) if AddValues: return uniqueId, 200 else: return 400
def register(): form1 = request.form if request.method == 'POST': acc = M.Account(form1['Fname'], form1['Lname'], form1['email'], form1['phone'], form1['username'], form1['password'], form1['oAuth']) db.insertAccount(acc) return render_template('signin.html', form=form1) return render_template('register.html', form=form1)
def post(self): parser.add_argument('Date', type=str) parser.add_argument('Valor', type=float) parser.add_argument('Tipo', type=str) args = parser.parse_args() received, uniqueId = Models.AddReceivedModels(args) AddValues = DbPostServices.SendReceived(received) if AddValues: return uniqueId, 200 else: return 400
def run_multi(self): from Models import Models model_list = [] for i in range(num_cores): model_list.append(smallModel(pow(10,-1))) models = Models(model_list) models.train() models.validation() models.dump() print("Test ended")
def post(self): parser.add_argument('CardName', type=str) parser.add_argument('Vencimento', type=int) parser.add_argument('Fechamento', type=int) # adicionando cartão de crédito args = parser.parse_args() DebtsValues, uniqueId = Models.AddCredCardModels(args) AddValues = DbPostServices.SendCredCard(DebtsValues) if AddValues: return uniqueId, 200 else: return 400
def post(self): parser.add_argument('CardName', type=str) parser.add_argument('Valor', type=float) parser.add_argument('QuantidadeDeParcelasCartao', type=str) parser.add_argument('TipoDeDividaCartao', type=str) parser.add_argument('DataCompra', type=str) parser.add_argument('Descricao', type=str) # adicionando valores ao cartão de crédito args = parser.parse_args() DebtsValues, uniqueId = Models.AddValuesCredCard(args) AddValues = DbPostServices.SendCardsValues(DebtsValues) if AddValues: return uniqueId, 200 else: return 400
def post(self): parser.add_argument('Name', type=str) parser.add_argument('Valor', type=float) parser.add_argument('Vencimento', type=str) parser.add_argument('QuantidadeParcelas', type=str) parser.add_argument('TipoDeDivida', type=str) # adicionando dívidas simples args = parser.parse_args() DebtsValues, uniqueId, result = Models.AddDebtsValuesModels(args) if result: AddValues = DbPostServices.SendDebtsValues(DebtsValues) else: return DebtsValues, 400 if AddValues: return uniqueId, 200 else: return 400
def execute(self): # parameters epsilon = .5 # exploration epsilon_decay = 0.95 epsilon_min = 0.1 epoch = 4000 # is number of cycles... max_memory = 2000 # NEEDS TO BE AS BIG AS AT LEAST 1 TRADING DAY!!! batch_size = 50 # 50 sequence_length = 250 # 500 discount = 0.95 training_days = 1 testing_days = 1 features_list = list(range(1, 33)) ## FULL features_list = list(range(1, 6)) ## SHORT!! training_store = ds.DataStore(training_days=training_days, features_list=features_list, sequence_length=sequence_length) features_length = training_store.get_features_length() env = Trading(data_store=training_store, sequence_length=sequence_length, features_length=features_length) num_actions = env.get_action_count( ) # [sell, buy, flat] # get From TRADING!! #testing_store = ds.DataStore(training_days=training_days, testing_days=10, features_list=features_list, sequence_length=sequence_length) mo = Models() rms = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-06) use_ufcnn = True if use_ufcnn: model = mo.model_ufcnn_concat(sequence_length=sequence_length, features=features_length, nb_filter=15, filter_length=5, output_dim=num_actions, optimizer=rms, loss='mse', batch_size=batch_size, init="normal") base_model_name = "ufcnn" else: model = mo.atari_conv_model(output_dim=num_actions, features=features_length, loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size, init="normal") base_model_name = "atari" testing_store = ds.DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=sequence_length, mean=training_store.mean, std=training_store.std) test_env = Trading(data_store=testing_store, sequence_length=sequence_length, features_length=features_length) #model = mo.atari_conv_model(regression=False, output_dim=num_actions, features=features_length, nb_filter=50, # loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size) # If you want to continue training from a previous model, just uncomment the line bellow #mo.load_model("ufcnn_rl_training") # Define environment/game # Initialize experience replay object start_time = time.time() best_pnl = -99999. best_rndless_pnl = -99999. exp_replay = ExperienceReplay(max_memory=max_memory, env=env, sequence_dim=(sequence_length, features_length), discount=discount) lineindex = 0 # Train for e in range(epoch): loss = 0. game_over = False total_reward = 0 win_cnt = 0 loss_cnt = 0 random_cnt = 0 no_random_cnt = 0 ### loop over days-... for i in range(training_days): input_t = env.reset() j = 0 while not game_over: # game_over ... end of trading day... input_tm1 = input_t #print("INPUT ",input_tm1) # get next action if np.random.rand() <= epsilon: action = np.random.randint(0, num_actions, size=1)[0] random_cnt += 1 #print("RANDOM") else: q = model.predict(exp_replay.resize_input(input_tm1)) action = np.argmax(q[0]) no_random_cnt += 1 #print("SELECT") ##action = np.argmax(q) # apply action, get rewards and new state input_t, reward, game_over, idays, lineindex = env.act( action) if reward > 0: win_cnt += 1 if reward < 0: loss_cnt += 1 total_reward += reward if reward > 1.: reward = 1. if reward < -1.: reward = -1. # store experience exp_replay.remember([action, reward, idays, lineindex - 1], game_over) # adapt model if j > batch_size: # do not run exp_rep if the store is empty... inputs, targets = exp_replay.get_batch( model, batch_size=batch_size) curr_loss = model.train_on_batch( exp_replay.resize_input(inputs), targets) loss += curr_loss j += 1 rndless_pnl = self.get_randomless_pnl(test_env=test_env, model=model, testing_days=testing_days) secs = time.time() - start_time print( "Epoch {:05d}/{} | Time {:7.1f} | Loss {:11.4f} | Win trades {:5d} | Loss trades {:5d} | Total PnL {:8.2f} | Rndless PnL {:8.2f} | Eps {:.4f} | Rnd: {:5d}| No Rnd: {:5d} " .format(e, epoch, secs, loss, win_cnt, loss_cnt, total_reward, rndless_pnl, epsilon, random_cnt, no_random_cnt), flush=True) if epsilon > epsilon_min: epsilon *= epsilon_decay # Save trained model weights and architecture, this will be used by the visualization code if total_reward > best_pnl: mo.save_model(model, base_model_name + "_rl_best") best_pnl = total_reward else: mo.save_model(model, base_model_name + "_rl_training") if rndless_pnl > best_pnl: mo.save_model(model, base_model_name + "_rl_rndless_best") best_rndless_pnl = rndless_pnl
training_days = 1 testing_days = 1 max_memory = 500000 training_store = ds.DataStore(training_days=training_days, features_list=features_list, sequence_length=sequence_length) testing_store = ds.DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=sequence_length, mean=training_store.mean, std=training_store.std) features_length = training_store.get_features_length() env = Trading(data_store=testing_store, sequence_length=sequence_length, features_length=features_length) num_actions = env.get_action_count() # [sell, buy, flat] # get From TRADING!! mo = Models() start_time = time.time() best_pnl = -99999. exp_replay = ExperienceReplay(max_memory=max_memory, env=env, sequence_dim=(sequence_length, features_length)) if len(sys.argv) == 2: model_name = sys.argv[1] else: model_name = None if model_name is not None: model = mo.load_model(model_name) model.compile(optimizer='rmsprop', loss='mse') else:
class Ui_MainWindow(object): def __init__(self): # super(self).__init__() # инициализация классификаторов self.models = Models() def setupUi(self, MainWindow): MainWindow.setObjectName("MainWindow") MainWindow.resize(1038, 593) self.centralwidget = QtWidgets.QWidget(MainWindow) self.centralwidget.setObjectName("centralwidget") self.gridLayoutWidget = QtWidgets.QWidget(self.centralwidget) self.gridLayoutWidget.setGeometry(QtCore.QRect(419, 240, 561, 261)) self.gridLayoutWidget.setObjectName("gridLayoutWidget") self.gridLayout = QtWidgets.QGridLayout(self.gridLayoutWidget) self.gridLayout.setContentsMargins(0, 0, 0, 0) self.gridLayout.setObjectName("gridLayout") # зона результатов self.label_nbc_type = QtWidgets.QLabel(self.gridLayoutWidget) self.label_nbc_type.setText("") self.label_nbc_type.setObjectName("label_nbc_type") self.label_nbc_type.setAlignment(QtCore.Qt.AlignCenter) self.label_nbc_type.setVisible(False) self.gridLayout.addWidget(self.label_nbc_type, 3, 1, 1, 1) self.label_svm_zone = QtWidgets.QLabel(self.gridLayoutWidget) self.label_svm_zone.setText("") self.label_svm_zone.setObjectName("label_svm_zone") self.label_svm_zone.setAlignment(QtCore.Qt.AlignCenter) self.label_svm_zone.setVisible(False) self.gridLayout.addWidget(self.label_svm_zone, 1, 2, 1, 1) self.label_knn_zone = QtWidgets.QLabel(self.gridLayoutWidget) self.label_knn_zone.setText("") self.label_knn_zone.setObjectName("label_knn_zone") self.label_knn_zone.setAlignment(QtCore.Qt.AlignCenter) self.label_knn_zone.setVisible(False) self.gridLayout.addWidget(self.label_knn_zone, 2, 2, 1, 1) self.label_nbc_zone = QtWidgets.QLabel(self.gridLayoutWidget) self.label_nbc_zone.setText("") self.label_nbc_zone.setObjectName("label_nbc_zone") self.label_nbc_zone.setAlignment(QtCore.Qt.AlignCenter) self.label_nbc_zone.setVisible(False) self.gridLayout.addWidget(self.label_nbc_zone, 3, 2, 1, 1) self.label_svm_head = QtWidgets.QLabel(self.gridLayoutWidget) self.label_svm_head.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_svm_head.setObjectName("label_svm_head") self.label_svm_head.setVisible(False) self.gridLayout.addWidget(self.label_svm_head, 1, 0, 1, 1) self.label_type_head = QtWidgets.QLabel(self.gridLayoutWidget) self.label_type_head.setObjectName("label_type_head") self.label_type_head.setAlignment(QtCore.Qt.AlignCenter) self.label_type_head.setVisible(False) self.gridLayout.addWidget(self.label_type_head, 0, 1, 1, 1) self.label_knn_head = QtWidgets.QLabel(self.gridLayoutWidget) self.label_knn_head.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_knn_head.setObjectName("label_knn_head") self.label_knn_head.setVisible(False) self.gridLayout.addWidget(self.label_knn_head, 2, 0, 1, 1) self.label_zone_head = QtWidgets.QLabel(self.gridLayoutWidget) self.label_zone_head.setObjectName("label_zone_head") self.label_zone_head.setAlignment(QtCore.Qt.AlignCenter) self.label_zone_head.setVisible(False) self.gridLayout.addWidget(self.label_zone_head, 0, 2, 1, 1) self.label_nbc_head = QtWidgets.QLabel(self.gridLayoutWidget) self.label_nbc_head.setAlignment(QtCore.Qt.AlignCenter) self.label_nbc_head.setObjectName("label_nbc_head") self.label_nbc_head.setVisible(False) self.gridLayout.addWidget(self.label_nbc_head, 3, 0, 1, 1) self.label_knn_type = QtWidgets.QLabel(self.gridLayoutWidget) self.label_knn_type.setText("") self.label_knn_type.setObjectName("label_knn_type") self.label_knn_type.setAlignment(QtCore.Qt.AlignCenter) self.label_knn_type.setVisible(False) self.gridLayout.addWidget(self.label_knn_type, 2, 1, 1, 1) self.label_svm_type = QtWidgets.QLabel(self.gridLayoutWidget) self.label_svm_type.setText("") self.label_svm_type.setObjectName("label_svm_type") self.label_svm_type.setAlignment(QtCore.Qt.AlignCenter) self.label_svm_type.setVisible(False) # зона вводов self.gridLayout.addWidget(self.label_svm_type, 1, 1, 1, 1) self.horizontalLayoutWidget_2 = QtWidgets.QWidget(self.centralwidget) self.horizontalLayoutWidget_2.setGeometry(QtCore.QRect(120, 110, 251, 451)) self.horizontalLayoutWidget_2.setObjectName("horizontalLayoutWidget_2") self.horizontalLayout_4 = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget_2) self.horizontalLayout_4.setContentsMargins(0, 0, 0, 0) self.horizontalLayout_4.setObjectName("horizontalLayout_4") self.verticalLayout = QtWidgets.QVBoxLayout() self.verticalLayout.setObjectName("verticalLayout") self.label_Al = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Al.setFont(font) self.label_Al.setTabletTracking(True) self.label_Al.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Al.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Al.setObjectName("label_Al") self.verticalLayout.addWidget(self.label_Al) self.label_Ba = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Ba.setFont(font) self.label_Ba.setTabletTracking(True) self.label_Ba.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Ba.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Ba.setObjectName("label_Ba") self.verticalLayout.addWidget(self.label_Ba) self.label_Ca = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Ca.setFont(font) self.label_Ca.setTabletTracking(True) self.label_Ca.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Ca.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Ca.setObjectName("label_Ca") self.verticalLayout.addWidget(self.label_Ca) self.label_Cu = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Cu.setFont(font) self.label_Cu.setTabletTracking(True) self.label_Cu.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Cu.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Cu.setObjectName("label_Cu") self.verticalLayout.addWidget(self.label_Cu) self.label_Fe = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Fe.setFont(font) self.label_Fe.setTabletTracking(True) self.label_Fe.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Fe.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Fe.setObjectName("label_Fe") self.verticalLayout.addWidget(self.label_Fe) self.label_K = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_K.setFont(font) self.label_K.setTabletTracking(True) self.label_K.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_K.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_K.setObjectName("label_K") self.verticalLayout.addWidget(self.label_K) self.label_Li = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Li.setFont(font) self.label_Li.setTabletTracking(True) self.label_Li.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Li.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Li.setObjectName("label_Li") self.verticalLayout.addWidget(self.label_Li) self.label_Mg = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Mg.setFont(font) self.label_Mg.setTabletTracking(True) self.label_Mg.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Mg.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Mg.setObjectName("label_Mg") self.verticalLayout.addWidget(self.label_Mg) self.label_Mn = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Mn.setFont(font) self.label_Mn.setTabletTracking(True) self.label_Mn.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Mn.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Mn.setObjectName("label_Mn") self.verticalLayout.addWidget(self.label_Mn) self.label_Na = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Na.setFont(font) self.label_Na.setTabletTracking(True) self.label_Na.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Na.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Na.setObjectName("label_Na") self.verticalLayout.addWidget(self.label_Na) self.label_Ni = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Ni.setFont(font) self.label_Ni.setTabletTracking(True) self.label_Ni.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Ni.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Ni.setObjectName("label_Ni") self.verticalLayout.addWidget(self.label_Ni) self.label_Rb = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Rb.setFont(font) self.label_Rb.setTabletTracking(True) self.label_Rb.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Rb.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Rb.setObjectName("label_Rb") self.verticalLayout.addWidget(self.label_Rb) self.label_Sr = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Sr.setFont(font) self.label_Sr.setTabletTracking(True) self.label_Sr.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Sr.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Sr.setObjectName("label_Sr") self.verticalLayout.addWidget(self.label_Sr) self.label_Ti = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Ti.setFont(font) self.label_Ti.setTabletTracking(True) self.label_Ti.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Ti.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Ti.setObjectName("label_Ti") self.verticalLayout.addWidget(self.label_Ti) self.label_Zn = QtWidgets.QLabel(self.horizontalLayoutWidget_2) font = QtGui.QFont() font.setPointSize(10) self.label_Zn.setFont(font) self.label_Zn.setTabletTracking(True) self.label_Zn.setLayoutDirection(QtCore.Qt.LeftToRight) self.label_Zn.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.label_Zn.setObjectName("label_Zn") self.verticalLayout.addWidget(self.label_Zn) self.horizontalLayout_4.addLayout(self.verticalLayout) self.verticalLayout_2 = QtWidgets.QVBoxLayout() self.verticalLayout_2.setObjectName("verticalLayout_2") self.lineEdit_Al = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Al.setObjectName("lineEdit_Al") self.verticalLayout_2.addWidget(self.lineEdit_Al) self.lineEdit_Ba = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Ba.setObjectName("lineEdit_Ba") self.verticalLayout_2.addWidget(self.lineEdit_Ba) self.lineEdit_Ca = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Ca.setObjectName("lineEdit_Ca") self.verticalLayout_2.addWidget(self.lineEdit_Ca) self.lineEdit_Cu = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Cu.setObjectName("lineEdit_Cu") self.verticalLayout_2.addWidget(self.lineEdit_Cu) self.lineEdit_Fe = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Fe.setObjectName("lineEdit_Fe") self.verticalLayout_2.addWidget(self.lineEdit_Fe) self.lineEdit_K = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_K.setObjectName("lineEdit_K") self.verticalLayout_2.addWidget(self.lineEdit_K) self.lineEdit_Li = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Li.setObjectName("lineEdit_Li") self.verticalLayout_2.addWidget(self.lineEdit_Li) self.lineEdit_Mg = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Mg.setObjectName("lineEdit_Mg") self.verticalLayout_2.addWidget(self.lineEdit_Mg) self.lineEdit_Mn = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Mn.setObjectName("lineEdit_Mn") self.verticalLayout_2.addWidget(self.lineEdit_Mn) self.lineEdit_Na = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Na.setObjectName("lineEdit_Na") self.verticalLayout_2.addWidget(self.lineEdit_Na) self.lineEdit_Ni = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Ni.setObjectName("lineEdit_Ni") self.verticalLayout_2.addWidget(self.lineEdit_Ni) self.lineEdit_Rb = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Rb.setObjectName("lineEdit_Rb") self.verticalLayout_2.addWidget(self.lineEdit_Rb) self.lineEdit_Sr = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Sr.setObjectName("lineEdit_Sr") self.verticalLayout_2.addWidget(self.lineEdit_Sr) self.lineEdit_Ti = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Ti.setObjectName("lineEdit_Ti") self.verticalLayout_2.addWidget(self.lineEdit_Ti) self.lineEdit_Zn = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2) self.lineEdit_Zn.setObjectName("lineEdit_Zn") self.verticalLayout_2.addWidget(self.lineEdit_Zn) self.horizontalLayout_4.addLayout(self.verticalLayout_2) self.label_input_head = QtWidgets.QLabel(self.centralwidget) self.label_input_head.setGeometry(QtCore.QRect(120, 90, 251, 16)) font = QtGui.QFont() font.setPointSize(12) self.label_input_head.setFont(font) self.label_input_head.setAlignment(QtCore.Qt.AlignCenter) self.label_input_head.setObjectName("label_input_head") self.verticalLayoutWidget_3 = QtWidgets.QWidget(self.centralwidget) self.verticalLayoutWidget_3.setGeometry(QtCore.QRect(119, 0, 251, 80)) self.verticalLayoutWidget_3.setObjectName("verticalLayoutWidget_3") self.verticalLayout_3 = QtWidgets.QVBoxLayout(self.verticalLayoutWidget_3) self.verticalLayout_3.setContentsMargins(0, 0, 0, 0) self.verticalLayout_3.setObjectName("verticalLayout_3") self.label_wt_head = QtWidgets.QLabel(self.verticalLayoutWidget_3) font = QtGui.QFont() font.setPointSize(12) self.label_wt_head.setFont(font) self.label_wt_head.setAlignment(QtCore.Qt.AlignCenter) self.label_wt_head.setObjectName("label_wt_head") self.verticalLayout_3.addWidget(self.label_wt_head) self.horizontalLayout_2 = QtWidgets.QHBoxLayout() self.horizontalLayout_2.setObjectName("horizontalLayout_2") self.radioButton_white = QtWidgets.QRadioButton(self.verticalLayoutWidget_3) self.radioButton_white.setChecked(True) self.radioButton_white.setObjectName("radioButton_white") self.horizontalLayout_2.addWidget(self.radioButton_white) self.radioButton_red = QtWidgets.QRadioButton(self.verticalLayoutWidget_3) self.radioButton_red.setLayoutDirection(QtCore.Qt.RightToLeft) self.radioButton_red.setChecked(False) self.radioButton_red.setObjectName("radioButton_red") self.horizontalLayout_2.addWidget(self.radioButton_red) self.verticalLayout_3.addLayout(self.horizontalLayout_2) self.label_results = QtWidgets.QLabel(self.centralwidget) self.label_results.setGeometry(QtCore.QRect(420, 180, 561, 51)) font = QtGui.QFont() font.setPointSize(12) self.label_results.setFont(font) self.label_results.setAlignment(QtCore.Qt.AlignCenter) self.label_results.setObjectName("label_results") self.label_results.setVisible(False) self.pushButton_calculate = QtWidgets.QPushButton(self.centralwidget) self.pushButton_calculate.setGeometry(QtCore.QRect(420, 530, 191, 28)) self.pushButton_calculate.setObjectName("pushButton_calculate") MainWindow.setCentralWidget(self.centralwidget) self.statusbar = QtWidgets.QStatusBar(MainWindow) self.statusbar.setObjectName("statusbar") MainWindow.setStatusBar(self.statusbar) self.pushButton_calculate.clicked.connect(self.calculate) self.retranslateUi(MainWindow) QtCore.QMetaObject.connectSlotsByName(MainWindow) def retranslateUi(self, MainWindow): MainWindow.setWindowTitle(QtWidgets.QApplication.translate("MainWindow", "Классификатор вин", None, -1)) self.label_svm_head.setText(QtWidgets.QApplication.translate("MainWindow", "Метод опорных векторов:", None, -1)) self.label_type_head.setText(QtWidgets.QApplication.translate("MainWindow", "Наименование", None, -1)) self.label_knn_head.setText( QtWidgets.QApplication.translate("MainWindow", "Метод k-ближайших соседей:", None, -1)) self.label_zone_head.setText(QtWidgets.QApplication.translate("MainWindow", "Зона", None, -1)) self.label_nbc_head.setText( QtWidgets.QApplication.translate("MainWindow", "Наивный Байесовский классификатор:", None, -1)) self.label_Al.setText(QtWidgets.QApplication.translate("MainWindow", "Al", None, -1)) self.label_Ba.setText(QtWidgets.QApplication.translate("MainWindow", "Ba", None, -1)) self.label_Ca.setText(QtWidgets.QApplication.translate("MainWindow", "Ca", None, -1)) self.label_Cu.setText(QtWidgets.QApplication.translate("MainWindow", "Cu", None, -1)) self.label_Fe.setText(QtWidgets.QApplication.translate("MainWindow", "Fe", None, -1)) self.label_K.setText(QtWidgets.QApplication.translate("MainWindow", "K", None, -1)) self.label_Li.setText(QtWidgets.QApplication.translate("MainWindow", "Li", None, -1)) self.label_Mg.setText(QtWidgets.QApplication.translate("MainWindow", "Mg", None, -1)) self.label_Mn.setText(QtWidgets.QApplication.translate("MainWindow", "Mn", None, -1)) self.label_Na.setText(QtWidgets.QApplication.translate("MainWindow", "Na", None, -1)) self.label_Ni.setText(QtWidgets.QApplication.translate("MainWindow", "Ni", None, -1)) self.label_Rb.setText(QtWidgets.QApplication.translate("MainWindow", "Rb", None, -1)) self.label_Sr.setText(QtWidgets.QApplication.translate("MainWindow", "Sr", None, -1)) self.label_Ti.setText(QtWidgets.QApplication.translate("MainWindow", "Ti", None, -1)) self.label_Zn.setText(QtWidgets.QApplication.translate("MainWindow", "Zn", None, -1)) self.label_input_head.setText(QtWidgets.QApplication.translate("MainWindow", "Элементный состав", None, -1)) self.label_wt_head.setText(QtWidgets.QApplication.translate("MainWindow", "Тип вина", None, -1)) self.radioButton_white.setText(QtWidgets.QApplication.translate("MainWindow", "Белое", None, -1)) self.radioButton_red.setText(QtWidgets.QApplication.translate("MainWindow", "Красное", None, -1)) self.label_results.setText(QtWidgets.QApplication.translate("MainWindow", "Результаты", None, -1)) self.pushButton_calculate.setText(QtWidgets.QApplication.translate("MainWindow", "Классифицировать", None, -1)) @QtCore.Slot() def calculate(self): self.label_results.setVisible(True) elements = ['Al', 'Ba', 'Ca', 'Cu', 'Fe', 'K', 'Li', 'Mg', 'Mn', 'Na', 'Ni', 'Rb', 'Sr', 'Ti', 'Zn'] element_values = list() for element in elements: element_values.append(float(convert_colons(eval('self.lineEdit_{}.text()'.format(element))))) element_values = np.array(element_values) if self.radioButton_white.isChecked(): wine_type = 'white' else: wine_type = 'red' ret_dict = self.models.predict(wine_type, element_values) for method in ['svm', 'knn', 'nbc']: exec('self.label_{}_head.setVisible(True)'.format(method)) for class_type in ['type', 'zone']: exec('self.label_{}_head.setVisible(True)'.format(class_type)) current_value = ret_dict['{}_{}'.format(method, class_type)][0] exec('self.label_{}_{}.setText(current_value)'.format(method, class_type)) exec('self.label_{}_{}.setVisible(True)'.format(method, class_type))
def test(): models = Models() exp = RegressionExperiment("med_random") exp.predict(FeaturePipeline.ngram_reg(models.get("svr"), 1))
if es.ping(): print('Connected to Elasticsearch') else: print('Could not connect to elasticsearch') sys.exit() def remove_special_chars(text): '''This function removes the special chars from the text''' text=str(text) text = re.sub('[^A-Za-z0-9]+', ' ', text) text=text.lower() return text #Loading the Universal sentence encoder model model=Models() def get_query_doc(query): query=remove_special_chars(query) query_vector =model.get_vec_rep(query) # Getting the vector representation of text from the model #Creating a document structure to search with query. query_doc = { "query" : { "script_score" : { "query" : { "match_all": {} }, "script" : { "source": "cosineSimilarity(params.query_vector, 'text_vector') + 1.0",
def test_makeModels(self): m1 = Models() self.assertEqual(len(m1.modelList), 6)
if args.method.lower() in ['sorecgatitem']: dataset = SocialItem_Dataset(args) elif args.method.lower() in ['sorecgatuser']: dataset = SocialUser_Dataset(args) else: dataset = Dataset(args) params = Parameters(args, dataset) print( """Load data done [%.1f s]. #user:%d, #item:%d, #dom:%d, #train:%d, #test:%d, #valid:%d""" % (time() - t1, params.num_users, params.num_items, params.num_doms, params.num_train_instances, params.num_test_instances, params.num_valid_instances)) print('Method: %s' % (params.method)) if params.method in ['sorecgatitem', 'sorecgatuser']: model = Models(params) model.define_model() model.define_loss('all') print("Model definition completed: in %.2fs" % (time() - t1)) train_step = get_optimizer(params.learn_rate, params.optimizer).minimize(model.loss) init = tf.global_variables_initializer() config = tf.ConfigProto() config.gpu_options.allow_growth = True print('train instances: {}'.format(params.train_matrix.nnz)) error_plot = Error_plot(save_flag=True, res_path=params.result_path, args_str=args_str, args=args)
config_file_path = results_folder + '/params.ini' args = train_utils.parse_params(config_file_path) L = args.num_classes args.image_dim = [128, 128, 3] #%% load saved network parameters and open new session tf.reset_default_graph() in_placeholder = tf.placeholder( tf.float32, shape=[None, None, None, L + args.image_dim[2]], name="in_placeholder") out_placeholder = tf.placeholder(tf.float32, shape=[None, None, None, L], name='out_placeholder') phase = tf.placeholder(tf.bool, name='phase') net_class = Models(args) net_class.build_model(in_placeholder, phase) sess = tf.Session() saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(checkpoints_path) if ckpt and ckpt.model_checkpoint_path: ckpt_path = checkpoints_path + 'my_model-' + str(checkpoint[0]) saver.restore(sess, ckpt_path) #%% if not os.path.exists(output_path): os.makedirs(output_path) image_list = sorted(os.listdir(pascal_path + '/images'))
class Unshred: def __init__(self, verbose = False): self.encrypted_string = ( "de| | f|Cl|nf|ed|au| i|ti| |ma|ha|or|nn|ou| S|on|nd|on" "\n" "ry| |is|th|is| b|eo|as| | |f |wh| o|ic| t|, | |he|h " "\n" "ab| |la|pr|od|ge|ob| m|an| |s |is|el|ti|ng|il|d |ua|c " "\n" "he| |ea|of|ho| m| t|et|ha| | t|od|ds|e |ki| c|t |ng|br" "\n" "wo|m,|to|yo|hi|ve|u | t|ob| |pr|d |s |us| s|ul|le|ol|e " "\n" ' t|ca| t|wi| M|d |th|"A|ma|l |he| p|at|ap|it|he|ti|le|er' "\n" 'ry|d |un|Th|" |io|eo|n,|is| |bl|f |pu|Co|ic| o|he|at|mm' "\n" "hi| | |in| | | t| | | | |ye| |ar| |s | | |. ") self.rows = 8 self.columns = 19 self.matrix = [ [ 0 for i in range(self.columns) ] for j in range(self.rows) ] self.combination = Combination(self.rows, self.columns) self.used_column_indices = [] self.verbose = verbose def iterate_lines(self, foo): return iter(foo.splitlines()) def parseIntoMatrix(self): i = 0 for line in self.iterate_lines(self.encrypted_string): row_values = line.split('|') for j, value in enumerate(row_values): if value != '': self.matrix[i][j] = value i += 1 def initModels(self): self.corp = Models() self.corp.createLetterModel() self.corp.createWordUniGramModel() def computeCombinationProbability(self, phrases): sum = 0 for phrase in phrases: sum += self.corp.getLetterProbabilities(phrase) sum += self.corp.getWordProbabilities(phrase) if self.verbose: print '{0:15f} |{1:s}|'.format(sum, phrases[0]) #print sum, phrases return sum def getBestCombination(self): max_prob = -100000 max_combination = '' max_index = 0 for index, pair in enumerate(self.matrix[0]): # Don't check columns we already added if index in self.used_column_indices: continue temp_combination = copy.deepcopy(self.combination) # Get column values for index new_column_values = [self.matrix[i][index] for i in range(self.rows)] temp_combination.appendColumn(new_column_values) phrases_to_check = temp_combination.returnPhrases() # Compute probabilities probability = self.computeCombinationProbability(phrases_to_check) if probability > max_prob: max_prob = probability max_combination = temp_combination max_index = index if self.verbose: print '->{0:13f} |{1:s}|'.format(max_prob, max_combination.returnFirstPhrase()) self.used_column_indices.append(max_index) return max_prob, max_combination def unshred(self): # Find uppercase for index, pair in enumerate(self.matrix[0]): if pair[0].isupper(): self.combination.appendColumn([self.matrix[i][index] for i in range(self.rows)]) self.used_column_indices.append(index) break # Find next best combinations and append them for i in range(1, 19): prob, combination = self.getBestCombination() #print prob, combination self.combination = combination def displayResult(self): print '\n The unencrypted message is:\n' self.combination.display() def execute(self): self.parseIntoMatrix() self.initModels() self.unshred() self.displayResult() return
dataset = Dataset(args) params = Parameters(args,dataset) print("""Load data done [%.1f s]. #user:%d, #list:%d, #item:%d, #train:%d, #valid:%d, #test:%d"""% (time() - t1, params.num_user, params.num_list, params.num_item,params.num_train_instances,params.num_valid_instances,params.num_test_instances)) args.args_str = params.get_args_to_string() t1 = time() print("args str: ",args.args_str) print("leng from list_items_list: ",len(utils.get_value_lists_as_list(params.list_items_dct))) print("leng from trainArrTriplets: ", len((params.trainArrTriplets[0]))) print("non-zero entries in train_matrix: ", params.train_matrix.nnz) # model-loss-optimizer defn ======================================================================= models = Models(params,device=device) model = models.get_model() if params.loss not in ['bpr']: #bpr criterion_li = torch.nn.BCELoss() #criterion_li = torch.nn.BCEWithLogitsLoss() ## new change made if params.optimizer == 'adam': optimizer_gnn = torch.optim.Adam(model.parameters(), lr=params.lr) optimizer_seq = torch.optim.Adam(model.parameters(), lr=params.lr) elif params.optimizer == 'rmsprop': optimizer_gnn = torch.optim.RMSprop(model.parameters(), lr=params.lr) optimizer_seq = torch.optim.RMSprop(model.parameters(), lr=params.lr) model.to(device) # training ======================================================================= ## param =============================
def initModels(self): self.corp = Models() self.corp.createLetterModel() self.corp.createWordUniGramModel()
def Classify(X, Y, cls, rep, k=5000): # Start moment Start_moment = time.time() title = 'Classificando com {} e {} k={}'.format(cls, rep, k) print(title) # Creating the K-fold cross validator K_fold = KFold(n_splits=10, shuffle=True) # Labels test_labels = np.array([], 'int32') test_pred = np.array([], 'int32') # Confusion Matrix confusion = np.array([[0, 0], [0, 0]]) # The test for train_indices, test_indices in K_fold.split(X): print('Running .... =)') X_train = [X[i] for i in train_indices] Y_train = [Y[i] for i in train_indices] X_test = [X[i] for i in test_indices] Y_test = [Y[i] for i in test_indices] train_x, train_y, test_x, test_y = Representations( ).get_representation(rep=rep, train_x=X_train, train_y=Y_train, test_x=X_test, test_y=Y_test, k=k, cat=None) # c = Counter(Y_train) # print(Counter(train_y)) # print({1:c.most_common(1)[0][1], 0:c.most_common(1)[0][1], 2:c.most_common(1)[0][1]}) sm = SMOTE(sampling_strategy='minority', random_state=None) # sm = SMOTE(sampling_strategy={1:c.most_common(1)[0][1], 0:c.most_common(1)[0][1], 2:c.most_common(1)[0][1]}, random_state=None) # print(len(train_y)) train_x, train_y = sm.fit_sample(train_x, train_y) # print(Counter(train_y)) test_labels = np.append(test_labels, Y_test) classifier = Models().get_classifier(cls) classifier.fit(train_x, train_y) # Train_Classifier(classifier, X_train, Y_train) pred = classifier.predict(test_x) test_pred = np.append(test_pred, pred) # print(test_y) # print(pred) confusion += confusion_matrix(test_y, pred) # report = classification_report(test_labels, test_pred, target_names=['Contrário', 'Favorável'] if plb =='polaridade' else ['neutro', 'opiniao']) report = classification_report(test_labels, test_pred, target_names=['no', 'yes']) print(report) print("Confusion matrix:") print(confusion) Finish_moment = time.time() tm = "It took " + str((Finish_moment - Start_moment)) + " seconds" print(tm)
def __init__(self): # super(self).__init__() # инициализация классификаторов self.models = Models()
class Processor(object): def __init__(self, seed=999, n_splits=10): self.seed = seed self.n_splits = n_splits import sys sys.path.append( r"C:\Users\Kelvin\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation" ) #sys.path.append(r"C:\Users\Kelvi\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation") from Models import Models self.models = Models() from sklearn.model_selection import GridSearchCV from sklearn.model_selection import TimeSeriesSplit tscv = TimeSeriesSplit(n_splits) from sklearn.neural_network import MLPClassifier from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.gaussian_process import GaussianProcessClassifier from sklearn.gaussian_process.kernels import RBF from sklearn.tree import DecisionTreeClassifier from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier from sklearn.naive_bayes import GaussianNB from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis #self.models.add_model(model = GridSearchCV(estimator=MLPClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Neural Net') #self.models.add_model(model = GridSearchCV(estimator=KNeighborsClassifier(), param_grid={}, cv=tscv), model_name = 'KNN') #self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='linear', random_state=seed), param_grid={}, cv=tscv), model_name = 'Linear SVM') #self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='rbf', random_state=seed), param_grid={}, cv=tscv), model_name = 'RBF SVM') #self.models.add_model(model = GridSearchCV(estimator=GaussianProcessClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Gaussian Process') #self.models.add_model(model = GridSearchCV(estimator=DecisionTreeClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Decision Tree') #self.models.add_model(model = GridSearchCV(estimator=RandomForestClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Random Forest') #self.models.add_model(model = GridSearchCV(estimator=AdaBoostClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'AdaBoost') #self.models.add_model(model = GridSearchCV(estimator=GaussianNB(), param_grid={}, cv=tscv), model_name = 'Naive Bayes') ##self.models.add_model(model = GridSearchCV(estimator=QuadraticDiscriminantAnalysis(), param_grid={}, cv=tscv), model_name = 'QDA') self.models.add_model(model=MLPClassifier(random_state=seed), model_name='Neural Net') self.models.add_model(model=KNeighborsClassifier(), model_name='KNN') self.models.add_model(model=SVC(kernel='linear', random_state=seed), model_name='Linear SVM') self.models.add_model(model=SVC(kernel='rbf', random_state=seed), model_name='RBF SVM') self.models.add_model( model=GaussianProcessClassifier(random_state=seed), model_name='Gaussian Process') self.models.add_model(model=DecisionTreeClassifier(random_state=seed), model_name='Decision Tree') self.models.add_model(model=RandomForestClassifier(random_state=seed), model_name='Random Forest') self.models.add_model(model=AdaBoostClassifier(random_state=seed), model_name='AdaBoost') self.models.add_model(model=GaussianNB(), model_name='Naive Bayes') #self.models.add_model(model = QuadraticDiscriminantAnalysis(), model_name = 'QDA') def set_dataset(self, interval=30, no_of_steps=1, window_size=10, interpolation_method='linear', train_start='2009-01-01', train_end='2009-12-31', valid_start='2010-01-01', valid_end='2010-12-31', test_start='2011-01-01', test_end='2011-12-31', dimensions=3): from DB import DB from Dataset import Dataset from Models import Models import matplotlib.pyplot as plt self.interval = interval self.no_of_steps = no_of_steps self.window_size = window_size self.interpolation_method = interpolation_method self.train_start = train_start self.train_end = train_end self.valid_start = valid_start self.valid_end = valid_end self.test_start = test_start self.test_end = test_end self.dimensions = dimensions db = DB(driver='{SQL Server}', server='ENVY15-NOTEBOOK\MSSQL2017', database='DBHKUDissertation', username='******', password='******') #db = DB(driver = '{SQL Server}', server = 'LAPTOP-194NACED\SQL2017', database = 'DBHKUDissertation', username = '******', password = '******') sql = """ select [TradingDatetime] , [Open] , [High] , [Low] , [Close] , [Volume] FROM [DBHKUDissertation].[dbo].[TableStock] where Ticker in ('sh000300') and Interval in (""" + str(interval) + """) """ #df = db.read_sql(sql_string = sql) self.dataset = Dataset(data=db.read_sql(sql_string=sql), no_of_intervals_per_day=240 / interval, no_of_steps=no_of_steps) #print(self.dataset.get_data().head()) #self.dataset.visualize(columns=['Close']) print('created dataset object') self.dataset.interpolate(method=interpolation_method) #print(self.dataset.get_data().head()) print('interpolated') self.dataset.derive_features(window_size) #print(self.dataset.get_data().to_string()) print('derived features') self.dataset.remove_na() print(self.dataset.get_data().head()) print('removed na') #print(self.dataset.get_data().head()) #dataset.get_data().head()['ReturnDummy'].unique() self.dataset.data_splitting(train_start=train_start, train_end=train_end, valid_start=valid_start, valid_end=valid_end, test_start=test_start, test_end=test_end) #print(self.dataset.get_train().head()) print('splitted data') self.dataset.avoid_look_ahead_bias() #print(self.dataset.get_train().head()) print('removed look ahead') #self.dataset.set_X(['MA' + str(window_size), 'MA' + str(window_size * 2), 'MA' + str(window_size * 3), 'MA' + str(window_size * 4), 'MA' + str(window_size * 5), 'BB' + str(window_size), 'BB' + str(window_size * 2), 'BB' + str(window_size * 3), 'RSI' + str(window_size), 'RSI' + str(window_size * 2), 'RSI' + str(window_size * 3), 'STOCHK' + str(window_size), 'STOCHK' + str(window_size * 2), 'STOCHK' + str(window_size * 3), 'STOCHKD' + str(window_size), 'STOCHKD' + str(window_size * 2), 'STOCHKD' + str(window_size * 3)]) self.dataset.set_X([ 'MA' + str(window_size), 'MA' + str(window_size * 2), 'MA' + str(window_size * 3), 'BB' + str(window_size), 'BB' + str(window_size * 2), 'BB' + str(window_size * 3), 'RSI' + str(window_size), 'RSI' + str(window_size * 2), 'RSI' + str(window_size * 3), 'STOCHK' + str(window_size), 'STOCHK' + str(window_size * 2), 'STOCHK' + str(window_size * 3), 'STOCHKD' + str(window_size), 'STOCHKD' + str(window_size * 2), 'STOCHKD' + str(window_size * 3) ]) #print(self.dataset.get_train().head()) print('set X features') self.dataset.set_y(['ReturnDummy']) print('set Y feature') self.dataset.set_X_train() #print(self.dataset.get_X_train().head()) print('set X_train') self.dataset.set_y_train() print('set y_train') self.dataset.set_X_valid() print('set X_valid') self.dataset.set_y_valid() print('set y_valid') self.dataset.set_X_test() print('set X_test') self.dataset.set_y_test() print('set y_test') self.dataset.normalization() print('normalization') self.dataset.dimension_reduction(n_components=dimensions) print('dimension reduction') self.dataset.print_train_test_period() def get_dataset(self): return self.dataset def get_models(self): return self.models def train_validate_test_models(self): from sklearn.utils import column_or_1d from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score, precision_recall_fscore_support, classification_report import matplotlib.pyplot as plt from sklearn.metrics import roc_curve plt.figure(1) for model_name, model in self.models.get_models().items(): print(model_name) #print(model) model.fit(self.dataset.get_X_train(), column_or_1d(self.dataset.get_y_train())) y_pred = model.predict(self.dataset.get_X_test()) #print(accuracy_score(self.dataset.get_y_test(), y_pred)) print(classification_report(self.dataset.get_y_test(), y_pred)) #print(model.best_estimator_) #print(model.best_score_) #print(model.best_params_) #print(model.cv_results_) fpr, tpr, _ = roc_curve(self.dataset.get_y_test(), y_pred) plt.plot(fpr, tpr, label=model_name) plt.plot([0, 1], [0, 1], 'k--') plt.xlabel('False positive rate') plt.ylabel('True positive rate') plt.title('ROC curve') plt.legend(loc='best') plt.show() def train_validate_test_model(self, model_name): from sklearn.utils import column_or_1d from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score, precision_recall_fscore_support, classification_report print(model_name) model = self.models.get_model(model_name=model_name) model.fit(self.dataset.get_X_train(), column_or_1d(self.dataset.get_y_train())) y_train_pred = model.predict(self.dataset.get_X_train()) y_valid_pred = model.predict(self.dataset.get_X_valid()) y_test_pred = model.predict(self.dataset.get_X_test()) print(confusion_matrix(self.dataset.get_y_train(), y_train_pred)) print(confusion_matrix(self.dataset.get_y_valid(), y_valid_pred)) print(confusion_matrix(self.dataset.get_y_test(), y_test_pred)) train_tn, train_fp, train_fn, train_tp = confusion_matrix( self.dataset.get_y_train(), y_train_pred).ravel() valid_tn, valid_fp, valid_fn, valid_tp = confusion_matrix( self.dataset.get_y_valid(), y_valid_pred).ravel() test_tn, test_fp, test_fn, test_tp = confusion_matrix( self.dataset.get_y_test(), y_test_pred).ravel() return accuracy_score( self.dataset.get_y_train(), y_train_pred ), f1_score(self.dataset.get_y_train(), y_train_pred), precision_score( self.dataset.get_y_train(), y_train_pred), recall_score( self.dataset.get_y_train(), y_train_pred ), train_tn, train_fp, train_fn, train_tp, accuracy_score( self.dataset.get_y_valid(), y_valid_pred), f1_score( self.dataset.get_y_valid(), y_valid_pred), precision_score(self.dataset.get_y_valid( ), y_valid_pred), recall_score( self.dataset .get_y_valid(), y_valid_pred ), valid_tn, valid_fp, valid_fn, valid_tp, accuracy_score( self.dataset.get_y_test(), y_test_pred), f1_score( self.dataset.get_y_test(), y_test_pred), precision_score( self.dataset.get_y_test(), y_test_pred), recall_score( self.dataset.get_y_test(), y_test_pred ), test_tn, test_fp, test_fn, test_tp #def objective_function(self # , interval = 30 # , no_of_steps = 1 # , window_size = 10 # , interpolation_method = 'linear' # , train_start = '2009-01-01' # , train_end = '2009-12-31' # , test_start = '2010-01-01' # , test_end = '2010-12-31' # , dimensions = 3 # , model_name = 'RBF SVM'): # self.set_dataset(interval = interval # , no_of_steps = no_of_steps # , window_size = window_size # , interpolation_method = interpolation_method # , train_start = train_start # , train_end = train_end # , test_start = test_start # , test_end = test_end # , dimensions = dimensions) # return self.train_validate_test_model(model_name) def exhaustive_grid_search(self, interval=30, min_no_of_steps=10, max_no_of_steps=101, no_of_steps_interval=5, min_window_size=5, max_window_size=100, window_size_interval=5, interpolation_method='linear', train_start='2009-01-01', train_end='2009-12-31', valid_start='2010-01-01', valid_end='2010-12-31', test_start='2011-01-01', test_end='2011-12-31', dimensions=3, model_name='RBF SVM'): import numpy as np import pandas as pd #import matplotlib.pyplot as plt #from mpl_toolkits.mplot3d import Axes3D #from matplotlib import cm self.model_name = model_name #print(min_no_of_steps) no_of_steps = np.concatenate([[1], np.arange(min_no_of_steps, max_no_of_steps + 1, no_of_steps_interval)]) window_size = np.arange(min_window_size, max_window_size + 1, window_size_interval) #print(no_of_steps) #print(window_size) X, Y = np.meshgrid(no_of_steps, window_size) #print(X) #print(Y) #print(type(X)) X_train_start_date = [] X_train_end_date = [] X_valid_start_date = [] X_valid_end_date = [] X_test_start_date = [] X_test_end_date = [] train_accuracy = np.zeros(X.shape) train_F1 = np.zeros(X.shape) train_precision = np.zeros(X.shape) train_recall = np.zeros(X.shape) train_true_false_ratio = np.zeros(X.shape) train_no_of_true = np.zeros(X.shape) train_no_of_false = np.zeros(X.shape) train_tn = np.zeros(X.shape) train_fp = np.zeros(X.shape) train_fn = np.zeros(X.shape) train_tp = np.zeros(X.shape) valid_accuracy = np.zeros(X.shape) valid_F1 = np.zeros(X.shape) valid_precision = np.zeros(X.shape) valid_recall = np.zeros(X.shape) valid_true_false_ratio = np.zeros(X.shape) valid_no_of_true = np.zeros(X.shape) valid_no_of_false = np.zeros(X.shape) valid_tn = np.zeros(X.shape) valid_fp = np.zeros(X.shape) valid_fn = np.zeros(X.shape) valid_tp = np.zeros(X.shape) test_accuracy = np.zeros(X.shape) test_F1 = np.zeros(X.shape) test_precision = np.zeros(X.shape) test_recall = np.zeros(X.shape) test_true_false_ratio = np.zeros(X.shape) test_no_of_true = np.zeros(X.shape) test_no_of_false = np.zeros(X.shape) test_tn = np.zeros(X.shape) test_fp = np.zeros(X.shape) test_fn = np.zeros(X.shape) test_tp = np.zeros(X.shape) seed = np.zeros(X.shape) n_splits = np.zeros(X.shape) rows = 0 #print(Z) for step_row, window_row in zip(X, Y): columns = 0 for step, window in zip(step_row, window_row): print(step) print(window) self.set_dataset(interval=interval, no_of_steps=step, window_size=window, interpolation_method=interpolation_method, train_start=train_start, train_end=train_end, valid_start=valid_start, valid_end=valid_end, test_start=test_start, test_end=test_end, dimensions=dimensions) train_accuracy[rows, columns], train_F1[rows, columns], train_precision[ rows, columns], train_recall[rows, columns], train_tn[ rows, columns], train_fp[rows, columns], train_fn[ rows, columns], train_tp[rows, columns], valid_accuracy[ rows, columns], valid_F1[rows, columns], valid_precision[ rows, columns], valid_recall[rows, columns], valid_tn[ rows, columns], valid_fp[rows, columns], valid_fn[ rows, columns], valid_tp[rows, columns], test_accuracy[ rows, columns], test_F1[ rows, columns], test_precision[ rows, columns], test_recall[ rows, columns], test_tn[ rows, columns], test_fp[ rows, columns], test_fn[ rows, columns], test_tp[ rows, columns] = self.train_validate_test_model( model_name ) train_no_of_true[rows, columns] = self.dataset.get_y_train_true() train_no_of_false[rows, columns] = self.dataset.get_y_train_false() train_true_false_ratio[ rows, columns] = self.dataset.get_y_train_ratio() valid_no_of_true[rows, columns] = self.dataset.get_y_valid_true() valid_no_of_false[rows, columns] = self.dataset.get_y_valid_false() valid_true_false_ratio[ rows, columns] = self.dataset.get_y_valid_ratio() test_no_of_true[rows, columns] = self.dataset.get_y_test_true() test_no_of_false[rows, columns] = self.dataset.get_y_test_false() test_true_false_ratio[ rows, columns] = self.dataset.get_y_test_ratio() X_train_start_date.append( self.dataset.get_X_train_start_date().strftime( "%Y-%m-%d %H:%M:%S")) X_train_end_date.append( self.dataset.get_X_train_end_date().strftime( "%Y-%m-%d %H:%M:%S")) X_valid_start_date.append( self.dataset.get_X_valid_start_date().strftime( "%Y-%m-%d %H:%M:%S")) X_valid_end_date.append( self.dataset.get_X_valid_end_date().strftime( "%Y-%m-%d %H:%M:%S")) X_test_start_date.append( self.dataset.get_X_test_start_date().strftime( "%Y-%m-%d %H:%M:%S")) X_test_end_date.append( self.dataset.get_X_test_end_date().strftime( "%Y-%m-%d %H:%M:%S")) seed[rows, columns] = self.seed n_splits[rows, columns] = self.n_splits #Z[rows, columns] = self.objective_function(interval = interval # , no_of_steps = step # , window_size = window # , interpolation_method = interpolation_method # , train_start = train_start # , train_end = train_end # , test_start = test_start # , test_end = test_end # , dimensions = dimensions # , model_name = model_name) columns = columns + 1 rows = rows + 1 print(train_accuracy) print(X.reshape(X.size, 1)[:, 0]) print(Y.reshape(Y.size, 1)[:, 0]) print(train_accuracy.reshape(train_accuracy.size, 1)[:, 0]) df = pd.DataFrame(data={ 'interpolation_method': interpolation_method, 'train_start': train_start, 'train_end': train_end, 'valid_start': valid_start, 'valid_end': valid_end, 'test_start': test_start, 'test_end': test_end, 'dimensions': dimensions, 'model_name': model_name, 'no_of_steps': X.reshape(X.size, 1)[:, 0], 'window_size': Y.reshape(Y.size, 1)[:, 0], 'train_accuracy_score': train_accuracy.reshape(train_accuracy.size, 1)[:, 0], 'train_F1_score': train_F1.reshape(train_F1.size, 1)[:, 0], 'train_precision_score': train_precision.reshape(train_precision.size, 1)[:, 0], 'train_recall_score': train_recall.reshape(train_recall.size, 1)[:, 0], 'train_roc_tn': train_tn.reshape(train_tn.size, 1)[:, 0], 'train_roc_fp': train_fp.reshape(train_fp.size, 1)[:, 0], 'train_roc_fn': train_fn.reshape(train_fn.size, 1)[:, 0], 'train_roc_tp': train_tp.reshape(train_tp.size, 1)[:, 0], 'y_train_ratio': train_true_false_ratio.reshape(train_true_false_ratio.size, 1)[:, 0], 'y_train_true': train_no_of_true.reshape(train_no_of_true.size, 1)[:, 0], 'y_train_false': train_no_of_false.reshape(train_no_of_false.size, 1)[:, 0], 'valid_accuracy_score': valid_accuracy.reshape(valid_accuracy.size, 1)[:, 0], 'valid_F1_score': valid_F1.reshape(valid_F1.size, 1)[:, 0], 'valid_precision_score': valid_precision.reshape(valid_precision.size, 1)[:, 0], 'valid_recall_score': valid_recall.reshape(valid_recall.size, 1)[:, 0], 'valid_roc_tn': valid_tn.reshape(valid_tn.size, 1)[:, 0], 'valid_roc_fp': valid_fp.reshape(valid_fp.size, 1)[:, 0], 'valid_roc_fn': valid_fn.reshape(valid_fn.size, 1)[:, 0], 'valid_roc_tp': valid_tp.reshape(valid_tp.size, 1)[:, 0], 'y_valid_ratio': valid_true_false_ratio.reshape(valid_true_false_ratio.size, 1)[:, 0], 'y_valid_true': valid_no_of_true.reshape(valid_no_of_true.size, 1)[:, 0], 'y_valid_false': valid_no_of_false.reshape(valid_no_of_false.size, 1)[:, 0], 'test_accuracy_score': test_accuracy.reshape(test_accuracy.size, 1)[:, 0], 'test_F1_score': test_F1.reshape(test_F1.size, 1)[:, 0], 'test_precision_score': test_precision.reshape(test_precision.size, 1)[:, 0], 'test_recall_score': test_recall.reshape(test_recall.size, 1)[:, 0], 'test_roc_tn': test_tn.reshape(test_tn.size, 1)[:, 0], 'test_roc_fp': test_fp.reshape(test_fp.size, 1)[:, 0], 'test_roc_fn': test_fn.reshape(test_fn.size, 1)[:, 0], 'test_roc_tp': test_tp.reshape(test_tp.size, 1)[:, 0], 'y_test_ratio': test_true_false_ratio.reshape(test_true_false_ratio.size, 1)[:, 0], 'y_test_true': test_no_of_true.reshape(test_no_of_true.size, 1)[:, 0], 'y_test_false': test_no_of_false.reshape(test_no_of_false.size, 1)[:, 0], 'x_train_start_date': X_train_start_date, 'x_train_end_date': X_train_end_date, 'x_valid_start_date': X_valid_start_date, 'x_valid_end_date': X_valid_end_date, 'x_test_start_date': X_test_start_date, 'x_test_end_date': X_test_end_date, 'seed': seed.reshape(seed.size, 1)[:, 0], 'n_splits': n_splits.reshape(n_splits.size, 1)[:, 0], 'interval': interval }, index=np.arange(1, train_accuracy.size + 1)) self.X = X self.Y = Y self.train_accuracy = train_accuracy self.train_F1 = train_F1 self.train_precision = train_precision self.train_recall = train_recall self.train_tn = train_tn self.train_fp = train_fp self.train_fn = train_fn self.train_tp = train_tp self.valid_accuracy = valid_accuracy self.valid_F1 = valid_F1 self.valid_precision = valid_precision self.valid_recall = valid_recall self.valid_tn = valid_tn self.valid_fp = valid_fp self.valid_fn = valid_fn self.valid_tp = valid_tp self.test_accuracy = test_accuracy self.test_F1 = test_F1 self.test_precision = test_precision self.test_recall = test_recall self.test_tn = test_tn self.test_fp = test_fp self.test_fn = test_fn self.test_tp = test_tp self.grid_search_ouput = df return X, Y, train_accuracy, train_F1, train_precision, train_recall, train_tn, train_fp, train_fn, train_tp, valid_accuracy, valid_F1, valid_precision, valid_recall, valid_tn, valid_fp, valid_fn, valid_tp, test_accuracy, test_F1, test_precision, test_recall, test_tn, test_fp, test_fn, test_tp, df def to_sql(self, table_name, if_exists): from DB import DB db = DB(driver='{SQL Server}', server='ENVY15-NOTEBOOK\MSSQL2017', database='DBHKUDissertation', username='******', password='******') #db = DB(driver = '{SQL Server}', server = 'LAPTOP-194NACED\SQL2017', database = 'DBHKUDissertation', username = '******', password = '******') db.to_sql(df=self.grid_search_ouput, table_name=table_name, if_exists=if_exists) def delete_table(self, sql_string): from DB import DB db = DB(driver='{SQL Server}', server='ENVY15-NOTEBOOK\MSSQL2017', database='DBHKUDissertation', username='******', password='******') #db = DB(driver = '{SQL Server}', server = 'LAPTOP-194NACED\SQL2017', database = 'DBHKUDissertation', username = '******', password = '******') db.delete_table(sql_string) def plot_grid_search(self, is_saved, save_path): import matplotlib.pyplot as plt from mpl_toolkits.mplot3d import Axes3D from matplotlib import cm fig = plt.figure(figsize=(15, 9)) #fig = plt.figure() ax = Axes3D(fig) surf = ax.plot_surface(self.X, self.Y, self.valid_accuracy, rstride=2, cstride=2, cmap=cm.coolwarm, linewidth=0.5, antialiased=True) ax.view_init(elev=75, azim=-50) ax.set_title('Steps vs Window Size Optimization\n, model = ' + str(self.model_name) + ', intervale = ' + str(self.interval) + ', dimensions = ' + str(self.dimensions) + '\n, train_start = ' + str(self.train_start) + ', train_end = ' + str(self.train_end) + ', valid_start = ' + str(self.valid_start) + ', valid_end = ' + str(self.valid_end) + ', test_start = ' + str(self.test_start) + ', test_end = ' + str(self.test_end)) ax.set_xlabel('x = no_of_steps') ax.set_ylabel('y = window_size') ax.set_zlabel('f(x, y) = valid_accuracy_score') fig.colorbar(surf, shrink=0.5, aspect=5) plt.grid(True) if is_saved == 1: #filename = save_path + "\" + str(self.model_name) + "_" + str(self.interval) + '_' + str(self.dimensions) + '_' + str(self.train_start) + '_' + str(self.train_end) + '_' + str(self.test_start) + '_' + str(self.test_end) + '.png' filename = save_path + '\\' + str(self.model_name) + "_" + str( self.interval) + '_' + str(self.dimensions) + '_' + str( self.train_start) + '_' + str(self.train_end) + '_' + str( self.valid_start) + '_' + str( self.valid_end) + '_' + str( self.test_start) + '_' + str( self.test_end) + '.png' print(filename) plt.savefig(fname=filename) else: plt.show() def exhaustive_grid_search_models(self, interval=30, min_no_of_steps=5, max_no_of_steps=301, no_of_steps_interval=5, min_window_size=5, max_window_size=51, window_size_interval=5, interpolation_method='linear', train_start='2009-01-01', train_end='2009-12-31', valid_start='2010-01-01', valid_end='2010-12-31', test_start='2011-01-01', test_end='2011-12-31', dimensions=3): for model_name, model in self.models.get_models().items(): print(model_name) X, Y, train_accuracy, train_F1, train_precision, train_recall, train_tn, train_fp, train_fn, train_tp, valid_accuracy, valid_F1, valid_precision, valid_recall, valid_tn, valid_fp, valid_fn, valid_tp, test_accuracy, test_F1, test_precision, test_recall, test_tn, test_fp, test_fn, test_tp, grid_search_df = self.exhaustive_grid_search( interval=interval, min_no_of_steps=min_no_of_steps, max_no_of_steps=max_no_of_steps, no_of_steps_interval=no_of_steps_interval, min_window_size=min_window_size, max_window_size=max_window_size, window_size_interval=window_size_interval, interpolation_method=interpolation_method, train_start=train_start, train_end=train_end, valid_start=valid_start, valid_end=valid_end, test_start=test_start, test_end=test_end, dimensions=dimensions, model_name=model_name) self.to_sql(table_name='GridSearchResult', if_exists='append') self.plot_grid_search( is_saved=1, save_path= r"C:\Users\Kelvin\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation\GridSearchImage" )
def getModels(client): saveTickDateInString = Properties.SAVE_TICKDATE_IN_STRING models = Models(client, saveTickDateInString=saveTickDateInString) return models
__author__ = 'Placinta' from Models import Models encrypted_string = "Esp qtcde nzyqpcpynp zy esp ezatn zq Lcetqtntlw Tyepwwtrpynp hld spwo le Olcexzfes Nzwwprp ty estd jplc".upper( ) new_string = '' count = len(encrypted_string) possibilities = [] corp = Models() corp.createWordUniGramModel() for i in range(1, 37): character_list = [] new_string = "" for j in range(0, count): if encrypted_string[j] == ' ': character_list.append(' ') else: character_list.append( chr((ord(encrypted_string[j]) - 0x41 + i) % 26 + 0x41)) new_string = "".join(character_list).lower() possibilities.append(new_string) #print "{0}:".format(i) max_prob = -10000 max_string = '' for string in possibilities: probability = corp.getWordProbability(string) if probability > max_prob: max_prob = probability max_string = string
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu test_mkdir(args.save_path) if args.modelname == 'mnist_2nn' or args.modelname == 'mnist_cnn': datasetname = 'mnist' with tf.variable_scope('inputs') as scope: inputsx = tf.placeholder(tf.float32, [None, 784]) inputsy = tf.placeholder(tf.float32, [None, 10]) elif args.modelname == 'cifar10_cnn': datasetname = 'cifar10' with tf.variable_scope('inputs') as scope: inputsx = tf.placeholder(tf.float32, [None, 24, 24, 3]) inputsy = tf.placeholder(tf.float32, [None, 10]) myModel = Models(args.modelname, inputsx) predict_label = tf.nn.softmax(myModel.outputs) with tf.variable_scope('loss') as scope: Cross_entropy = -tf.reduce_mean(inputsy * tf.log(predict_label), axis=1) with tf.variable_scope('train') as scope: optimizer = tf.train.GradientDescentOptimizer(args.learning_rate) train = optimizer.minimize(Cross_entropy) with tf.variable_scope('validation') as scope: correct_prediction = tf.equal(tf.argmax(predict_label, axis=1), tf.argmax(inputsy, axis=1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) saver = tf.train.Saver(max_to_keep=3)
def execute(self): # parameters epsilon = .5 # exploration epsilon_decay = 0.95 epsilon_min = 0.1 epoch = 4000 # is number of cycles... max_memory = 2000 # NEEDS TO BE AS BIG AS AT LEAST 1 TRADING DAY!!! batch_size = 50 # 50 sequence_length = 250 # 500 discount = 0.95 training_days = 1 testing_days = 1 features_list = list(range(1,33)) ## FULL features_list = list(range(1,6)) ## SHORT!! training_store = ds.DataStore(training_days=training_days, features_list=features_list, sequence_length=sequence_length) features_length = training_store.get_features_length() env = Trading(data_store=training_store, sequence_length=sequence_length, features_length=features_length) num_actions = env.get_action_count() # [sell, buy, flat] # get From TRADING!! #testing_store = ds.DataStore(training_days=training_days, testing_days=10, features_list=features_list, sequence_length=sequence_length) mo = Models() rms = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-06) use_ufcnn=True if use_ufcnn: model = mo.model_ufcnn_concat(sequence_length=sequence_length, features=features_length, nb_filter=15, filter_length=5, output_dim=num_actions, optimizer=rms, loss='mse', batch_size = batch_size, init="normal") base_model_name = "ufcnn" else: model = mo.atari_conv_model(output_dim=num_actions, features=features_length, loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size, init="normal") base_model_name = "atari" testing_store = ds.DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=sequence_length, mean=training_store.mean, std=training_store.std) test_env = Trading(data_store=testing_store, sequence_length=sequence_length, features_length=features_length) #model = mo.atari_conv_model(regression=False, output_dim=num_actions, features=features_length, nb_filter=50, # loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size) # If you want to continue training from a previous model, just uncomment the line bellow #mo.load_model("ufcnn_rl_training") # Define environment/game # Initialize experience replay object start_time = time.time() best_pnl = -99999. best_rndless_pnl = -99999. exp_replay = ExperienceReplay(max_memory=max_memory, env=env, sequence_dim=(sequence_length, features_length), discount=discount) lineindex = 0 # Train for e in range(epoch): loss = 0. game_over = False total_reward = 0 win_cnt = 0 loss_cnt = 0 random_cnt = 0 no_random_cnt = 0 ### loop over days-... for i in range(training_days): input_t = env.reset() j = 0 while not game_over: # game_over ... end of trading day... input_tm1 = input_t #print("INPUT ",input_tm1) # get next action if np.random.rand() <= epsilon: action = np.random.randint(0, num_actions, size=1)[0] random_cnt += 1 #print("RANDOM") else: q = model.predict(exp_replay.resize_input(input_tm1)) action = np.argmax(q[0]) no_random_cnt += 1 #print("SELECT") ##action = np.argmax(q) # apply action, get rewards and new state input_t, reward, game_over, idays, lineindex = env.act(action) if reward > 0: win_cnt += 1 if reward < 0: loss_cnt += 1 total_reward += reward if reward > 1.: reward = 1. if reward < -1.: reward = -1. # store experience exp_replay.remember([action, reward, idays, lineindex-1], game_over) # adapt model if j > batch_size : # do not run exp_rep if the store is empty... inputs, targets = exp_replay.get_batch(model, batch_size=batch_size) curr_loss = model.train_on_batch(exp_replay.resize_input(inputs), targets) loss += curr_loss j += 1 rndless_pnl = self.get_randomless_pnl(test_env=test_env, model=model, testing_days=testing_days) secs = time.time() - start_time print("Epoch {:05d}/{} | Time {:7.1f} | Loss {:11.4f} | Win trades {:5d} | Loss trades {:5d} | Total PnL {:8.2f} | Rndless PnL {:8.2f} | Eps {:.4f} | Rnd: {:5d}| No Rnd: {:5d} ".format(e, epoch, secs, loss, win_cnt, loss_cnt, total_reward, rndless_pnl, epsilon, random_cnt, no_random_cnt), flush=True) if epsilon > epsilon_min: epsilon *= epsilon_decay # Save trained model weights and architecture, this will be used by the visualization code if total_reward > best_pnl: mo.save_model(model,base_model_name + "_rl_best") best_pnl = total_reward else: mo.save_model(model,base_model_name + "_rl_training") if rndless_pnl > best_pnl: mo.save_model(model,base_model_name + "_rl_rndless_best") best_rndless_pnl = rndless_pnl
testing_store = ds.DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=sequence_length, mean=training_store.mean, std=training_store.std) features_length = training_store.get_features_length() env = Trading(data_store=testing_store, sequence_length=sequence_length, features_length=features_length) num_actions = env.get_action_count( ) # [sell, buy, flat] # get From TRADING!! mo = Models() start_time = time.time() best_pnl = -99999. exp_replay = ExperienceReplay(max_memory=max_memory, env=env, sequence_dim=(sequence_length, features_length)) if len(sys.argv) == 2: model_name = sys.argv[1] else: model_name = None if model_name is not None: model = mo.load_model(model_name)
np.shape(x_train)[3] - args.num_classes) if args.app == 'seg': (x_test, seg_test) = (f_HDF5["test"], f_HDF5["test_gt"]) app_loss = Segmentation_Loss(args) elif args.app == 'dehaze': (x_test, gt_test) = (f_HDF5["test"], f_HDF5["test_gt"]) app_loss = Dehazing_Loss(args) elif args.app == 'matte': (x_test, x_test_1, x_test_2, gt_test) = (f_HDF5["val"], f_HDF5["test_t2"], f_HDF5["test_t1"], f_HDF5["test_gt"]) app_loss = Matting_Loss(args) print("Constructing network...") net_class = Models(args) tf.reset_default_graph() input_placeholder = tf.placeholder(tf.float32, (None, ) + x_train.shape[1:], 'images_with_seeds') phase = tf.placeholder(tf.bool, name='phase') output_from_network = net_class.build_model(input_placeholder, phase) loss = app_loss._loss(input_placeholder, output_from_network) samples_in_epoch = int( np.ceil(np.shape(x_train)[0] / np.float32(args.batch_size))) num_iter = args.num_epochs * samples_in_epoch + 1 global_step = tf.Variable(0, trainable=False) if args.train_schedule == 'exp': # decrease learning exponentially learning_rate = tf.train.exponential_decay(args.init_lr, global_step,samples_in_epoch*args.decay_every,\