def __init__(self, seed=999, n_splits=10):
self.seed = seed
self.n_splits = n_splits
import sys
sys.path.append(
r"C:\Users\Kelvin\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation"
)
#sys.path.append(r"C:\Users\Kelvi\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation")
from Models import Models
self.models = Models()
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import TimeSeriesSplit
tscv = TimeSeriesSplit(n_splits)
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
#self.models.add_model(model = GridSearchCV(estimator=MLPClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Neural Net')
#self.models.add_model(model = GridSearchCV(estimator=KNeighborsClassifier(), param_grid={}, cv=tscv), model_name = 'KNN')
#self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='linear', random_state=seed), param_grid={}, cv=tscv), model_name = 'Linear SVM')
#self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='rbf', random_state=seed), param_grid={}, cv=tscv), model_name = 'RBF SVM')
#self.models.add_model(model = GridSearchCV(estimator=GaussianProcessClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Gaussian Process')
#self.models.add_model(model = GridSearchCV(estimator=DecisionTreeClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Decision Tree')
#self.models.add_model(model = GridSearchCV(estimator=RandomForestClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Random Forest')
#self.models.add_model(model = GridSearchCV(estimator=AdaBoostClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'AdaBoost')
#self.models.add_model(model = GridSearchCV(estimator=GaussianNB(), param_grid={}, cv=tscv), model_name = 'Naive Bayes')
##self.models.add_model(model = GridSearchCV(estimator=QuadraticDiscriminantAnalysis(), param_grid={}, cv=tscv), model_name = 'QDA')
self.models.add_model(model=MLPClassifier(random_state=seed),
model_name='Neural Net')
self.models.add_model(model=KNeighborsClassifier(), model_name='KNN')
self.models.add_model(model=SVC(kernel='linear', random_state=seed),
model_name='Linear SVM')
self.models.add_model(model=SVC(kernel='rbf', random_state=seed),
model_name='RBF SVM')
self.models.add_model(
model=GaussianProcessClassifier(random_state=seed),
model_name='Gaussian Process')
self.models.add_model(model=DecisionTreeClassifier(random_state=seed),
model_name='Decision Tree')
self.models.add_model(model=RandomForestClassifier(random_state=seed),
model_name='Random Forest')
self.models.add_model(model=AdaBoostClassifier(random_state=seed),
model_name='AdaBoost')
self.models.add_model(model=GaussianNB(), model_name='Naive Bayes')
def train_model(new_data):
#declare objects
Data_preparation = data_preparation()
models = Models()
if new_data:
#read_data
data = Data_preparation.read_data_add_labels()
add_article_topic_col(data)
data = Data_preparation.add_full_text(data)
data = Data_preparation.add_binary_topics_col(data)
data.to_csv('new_data/new_processed_data.csv')
else:
data = pd.read_csv('new_data/new_processed_data.csv', index_col=0)
#for fast debug
#data = data.sample(n=1000)
train, test = train_test_split(data, test_size=0.1)
train1, train2 = train_test_split(train, test_size=0.5)
#train naive baise model
nb_model_obj = models.train_NB_model(train1)
zero_one_train_matrix = Data_preparation.create_zero_one_matrix(
nb_model_obj, train2)
lr_model_obj = models.train_lr_model(zero_one_train_matrix,
train2['LABEL'])
#save model
if save_model:
nb_pkl_filename = 'nb_pickle_model.pkl'
with open(nb_pkl_filename, 'wb') as file:
pickle.dump(nb_model_obj, file)
lr_pkl_filename = 'lr_pickle_model.pkl'
with open(lr_pkl_filename, 'wb') as file:
pickle.dump(lr_model_obj, file)
predict_obj = Predict(nb_model_obj, lr_model_obj)
nb_prediction = predict_obj.nb_predict(test, data_preparation)
print('test nb score: ' + str(np.mean(nb_prediction == test['LABEL'])))
lr_proba, lr_prediction = predict_obj.lr_predict(test, Data_preparation)
print('test lr score: ' + str(np.mean(lr_prediction == test['LABEL'])))
predict_obj.get_confusion_matrix(test['LABEL'], lr_prediction, 'all')
quantile_data, quantile_accurate = predict_obj.get_quantile_accurate(
test, lr_prediction, lr_proba)
with pd.option_context('display.max_rows', None, 'display.max_columns',
None):
print(quantile_accurate)
#todo add confusion matrix for each band
for index, row in quantile_accurate.iterrows():
print(row['probaBand'])
quantile = quantile_data[quantile_data['probaBand'] ==
row['probaBand']]
def test_challenge5(self):
self.driver.get("https://www.copart.com")
s = CopartSearchBar(self.driver)
model = "porsche"
s.search_input(model)
entryNumber100 = self.driver.find_element(
By.XPATH, "//*[@id='serverSideDataTable_length']//option[3]")
entryNumber100.click()
WebDriverWait(self.driver, 10).until(
expected_conditions.visibility_of_element_located(
(By.XPATH, "//*[@id='serverSideDataTable']/tbody/tr[100]")))
m = Models(self.driver)
m.unique_model_counter(100)
d = Damages(self.driver)
d.damage_finder(100)
def __init__(self, formal, caps, group):
# Load the dataset
data = DatasetLoader()
# Create and train the models
modelz = Models(data)
# modelz.showPerformances()
modelz.setSingleModel() # this will set the multiNB model
# Wait for the models to finish loading
while not modelz.endLoading:
time.sleep(1)
# Initialize the chat and run the dialogs
chat = ChatManager(modelz, group, formal, caps)
chat.run()
def Classify(X, Y, cls, rep, k=5000):
# Start moment
Start_moment = time.time()
title = 'Classificando com {} e {} k={}'.format(cls, rep, k)
print(title)
# Creating the K-fold cross validator
if 'w2v' in rep:
train_x = load(open('w2v_rep/{}_train_x.pkl'.format(rep), 'rb'))
train_y = load(open('w2v_rep/{}_train_y.pkl'.format(rep), 'rb'))
test_x = load(open('w2v_rep/{}_test_x.pkl'.format(rep), 'rb'))
test_y = load(open('w2v_rep/{}_test_y.pkl'.format(rep), 'rb'))
else:
X_train, X_test, y_train, y_test = train_test_split(X,
Y,
test_size=0.2,
random_state=123,
stratify=Y)
train_x, train_y, test_x, test_y = Representations(
).get_representation(rep=rep,
train_x=X_train,
train_y=y_train,
test_x=X_test,
test_y=y_test,
k=k,
cat=None)
sm = SMOTE(sampling_strategy='minority', random_state=None)
train_x, train_y = sm.fit_sample(train_x, train_y)
# dump(train_x, open('w2v_rep/{}_train_x.pkl'.format(rep), 'wb'))
# dump(train_y, open('w2v_rep/{}_train_y.pkl'.format(rep), 'wb'))
# dump(test_x, open('w2v_rep/{}_test_x.pkl'.format(rep), 'wb'))
# dump(test_y, open('w2v_rep/{}_test_y.pkl'.format(rep), 'wb'))
# return
classifier = Models().get_classifier(cls)
classifier.fit(train_x, train_y)
# Train_Classifier(classifier, X_train, Y_train)
pred = classifier.predict(test_x)
# report = classification_report(test_labels, test_pred, target_names=['Contrário', 'Favorável'] if plb =='polaridade' else ['neutro', 'opiniao'])
report = classification_report(test_y, pred, target_names=['no', 'yes'])
print(report)
Finish_moment = time.time()
tm = "It took " + str((Finish_moment - Start_moment)) + " seconds"
print(tm)
def event_details():
name = [
"Ed-Sheeran Concert", "MAGA Rally", "Chug-the-Jug", "Cupid's Ball",
"Deep Blue Destiny", "TGIF-Midnight", "Mexican-Fiesta",
"Paradise Awaits", "Frosty Frozen Prom", "Tequila Sunrise",
"Light Up the Night", "Zombie Prom"
]
date = [
"5-Oct-2018", "9-Oct-2018", "26-Oct-2018", "8-Nov-2018", "18-Nov-2018",
"4-Nov-2018", "21-Nov-2018", "26-Dec-2018", "8-Jan-2019"
]
price = [80, 100, 60, 150, 30, 45, 90, 200]
location = [
"Nashville, TN", "Memphis, TN", "Huntsville, AL", "New York, NY",
"Salt Lake City, UT", "Denver, CO"
]
events = []
for i in range(0, len(name)):
events.append(
M.Event(name[i], date[random.randint(0,
len(date) - 1)],
location[random.randint(0,
len(location) - 1)],
price[random.randint(0,
len(price) - 1)]))
return events
def __init__(self, modelType=None, numClass=None, numCell=None, maxLen=None, test=False):
if not test:
self.model = Models.getModel(modelType, numClass, numCell, maxLen).cuda()
self.criterion = nn.CrossEntropyLoss()
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0001)
else:
self.model = None
def retrieveAccount(username):
sql = """
select * from Account where username = '******'
""" % username
record = pd.read_sql(sql, engine)
returnValue = M.Account(record['FName'][0], record['LName'][0],
record['email'][0], record['phone'][0],
record['UserName'][0], record['pword'][0],
record['OAuth'][0])
return returnValue
def post(self):
# adicionando dívidas simples
args = parser.parse_args()
DebtsValues, uniqueId = Models.AddDebtsValuesModels(args)
AddValues = SendDbServices.SendSimpleDebts(connection,
DebtsValues[uniqueId])
if AddValues:
return uniqueId, 200
else:
return 400
def register():
form1 = request.form
if request.method == 'POST':
acc = M.Account(form1['Fname'], form1['Lname'], form1['email'],
form1['phone'], form1['username'], form1['password'],
form1['oAuth'])
db.insertAccount(acc)
return render_template('signin.html', form=form1)
return render_template('register.html', form=form1)
def post(self):
parser.add_argument('Date', type=str)
parser.add_argument('Valor', type=float)
parser.add_argument('Tipo', type=str)
args = parser.parse_args()
received, uniqueId = Models.AddReceivedModels(args)
AddValues = DbPostServices.SendReceived(received)
if AddValues:
return uniqueId, 200
else:
return 400
def run_multi(self):
from Models import Models
model_list = []
for i in range(num_cores):
model_list.append(smallModel(pow(10,-1)))
models = Models(model_list)
models.train()
models.validation()
models.dump()
print("Test ended")
def post(self):
parser.add_argument('CardName', type=str)
parser.add_argument('Vencimento', type=int)
parser.add_argument('Fechamento', type=int)
# adicionando cartão de crédito
args = parser.parse_args()
DebtsValues, uniqueId = Models.AddCredCardModels(args)
AddValues = DbPostServices.SendCredCard(DebtsValues)
if AddValues:
return uniqueId, 200
else:
return 400
def post(self):
parser.add_argument('CardName', type=str)
parser.add_argument('Valor', type=float)
parser.add_argument('QuantidadeDeParcelasCartao', type=str)
parser.add_argument('TipoDeDividaCartao', type=str)
parser.add_argument('DataCompra', type=str)
parser.add_argument('Descricao', type=str)
# adicionando valores ao cartão de crédito
args = parser.parse_args()
DebtsValues, uniqueId = Models.AddValuesCredCard(args)
AddValues = DbPostServices.SendCardsValues(DebtsValues)
if AddValues:
return uniqueId, 200
else:
return 400
def post(self):
parser.add_argument('Name', type=str)
parser.add_argument('Valor', type=float)
parser.add_argument('Vencimento', type=str)
parser.add_argument('QuantidadeParcelas', type=str)
parser.add_argument('TipoDeDivida', type=str)
# adicionando dívidas simples
args = parser.parse_args()
DebtsValues, uniqueId, result = Models.AddDebtsValuesModels(args)
if result:
AddValues = DbPostServices.SendDebtsValues(DebtsValues)
else:
return DebtsValues, 400
if AddValues:
return uniqueId, 200
else:
return 400
def execute(self):
# parameters
epsilon = .5 # exploration
epsilon_decay = 0.95
epsilon_min = 0.1
epoch = 4000 # is number of cycles...
max_memory = 2000 # NEEDS TO BE AS BIG AS AT LEAST 1 TRADING DAY!!!
batch_size = 50 # 50
sequence_length = 250 # 500
discount = 0.95
training_days = 1
testing_days = 1
features_list = list(range(1, 33)) ## FULL
features_list = list(range(1, 6)) ## SHORT!!
training_store = ds.DataStore(training_days=training_days,
features_list=features_list,
sequence_length=sequence_length)
features_length = training_store.get_features_length()
env = Trading(data_store=training_store,
sequence_length=sequence_length,
features_length=features_length)
num_actions = env.get_action_count(
) # [sell, buy, flat] # get From TRADING!!
#testing_store = ds.DataStore(training_days=training_days, testing_days=10, features_list=features_list, sequence_length=sequence_length)
mo = Models()
rms = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-06)
use_ufcnn = True
if use_ufcnn:
model = mo.model_ufcnn_concat(sequence_length=sequence_length,
features=features_length,
nb_filter=15,
filter_length=5,
output_dim=num_actions,
optimizer=rms,
loss='mse',
batch_size=batch_size,
init="normal")
base_model_name = "ufcnn"
else:
model = mo.atari_conv_model(output_dim=num_actions,
features=features_length,
loss='mse',
sequence_length=sequence_length,
optimizer=rms,
batch_size=batch_size,
init="normal")
base_model_name = "atari"
testing_store = ds.DataStore(training_days=training_days,
testing_days=testing_days,
features_list=features_list,
sequence_length=sequence_length,
mean=training_store.mean,
std=training_store.std)
test_env = Trading(data_store=testing_store,
sequence_length=sequence_length,
features_length=features_length)
#model = mo.atari_conv_model(regression=False, output_dim=num_actions, features=features_length, nb_filter=50,
# loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size)
# If you want to continue training from a previous model, just uncomment the line bellow
#mo.load_model("ufcnn_rl_training")
# Define environment/game
# Initialize experience replay object
start_time = time.time()
best_pnl = -99999.
best_rndless_pnl = -99999.
exp_replay = ExperienceReplay(max_memory=max_memory,
env=env,
sequence_dim=(sequence_length,
features_length),
discount=discount)
lineindex = 0
# Train
for e in range(epoch):
loss = 0.
game_over = False
total_reward = 0
win_cnt = 0
loss_cnt = 0
random_cnt = 0
no_random_cnt = 0
### loop over days-...
for i in range(training_days):
input_t = env.reset()
j = 0
while not game_over: # game_over ... end of trading day...
input_tm1 = input_t
#print("INPUT ",input_tm1)
# get next action
if np.random.rand() <= epsilon:
action = np.random.randint(0, num_actions, size=1)[0]
random_cnt += 1
#print("RANDOM")
else:
q = model.predict(exp_replay.resize_input(input_tm1))
action = np.argmax(q[0])
no_random_cnt += 1
#print("SELECT")
##action = np.argmax(q)
# apply action, get rewards and new state
input_t, reward, game_over, idays, lineindex = env.act(
action)
if reward > 0:
win_cnt += 1
if reward < 0:
loss_cnt += 1
total_reward += reward
if reward > 1.:
reward = 1.
if reward < -1.:
reward = -1.
# store experience
exp_replay.remember([action, reward, idays, lineindex - 1],
game_over)
# adapt model
if j > batch_size: # do not run exp_rep if the store is empty...
inputs, targets = exp_replay.get_batch(
model, batch_size=batch_size)
curr_loss = model.train_on_batch(
exp_replay.resize_input(inputs), targets)
loss += curr_loss
j += 1
rndless_pnl = self.get_randomless_pnl(test_env=test_env,
model=model,
testing_days=testing_days)
secs = time.time() - start_time
print(
"Epoch {:05d}/{} | Time {:7.1f} | Loss {:11.4f} | Win trades {:5d} | Loss trades {:5d} | Total PnL {:8.2f} | Rndless PnL {:8.2f} | Eps {:.4f} | Rnd: {:5d}| No Rnd: {:5d} "
.format(e, epoch, secs, loss, win_cnt, loss_cnt, total_reward,
rndless_pnl, epsilon, random_cnt, no_random_cnt),
flush=True)
if epsilon > epsilon_min:
epsilon *= epsilon_decay
# Save trained model weights and architecture, this will be used by the visualization code
if total_reward > best_pnl:
mo.save_model(model, base_model_name + "_rl_best")
best_pnl = total_reward
else:
mo.save_model(model, base_model_name + "_rl_training")
if rndless_pnl > best_pnl:
mo.save_model(model, base_model_name + "_rl_rndless_best")
best_rndless_pnl = rndless_pnl
training_days = 1
testing_days = 1
max_memory = 500000
training_store = ds.DataStore(training_days=training_days, features_list=features_list, sequence_length=sequence_length)
testing_store = ds.DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=sequence_length, mean=training_store.mean, std=training_store.std)
features_length = training_store.get_features_length()
env = Trading(data_store=testing_store, sequence_length=sequence_length, features_length=features_length)
num_actions = env.get_action_count() # [sell, buy, flat] # get From TRADING!!
mo = Models()
start_time = time.time()
best_pnl = -99999.
exp_replay = ExperienceReplay(max_memory=max_memory, env=env, sequence_dim=(sequence_length, features_length))
if len(sys.argv) == 2:
model_name = sys.argv[1]
else:
model_name = None
if model_name is not None:
model = mo.load_model(model_name)
model.compile(optimizer='rmsprop', loss='mse')
else:
class Ui_MainWindow(object):
def __init__(self):
# super(self).__init__()
# инициализация классификаторов
self.models = Models()
def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(1038, 593)
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
self.gridLayoutWidget = QtWidgets.QWidget(self.centralwidget)
self.gridLayoutWidget.setGeometry(QtCore.QRect(419, 240, 561, 261))
self.gridLayoutWidget.setObjectName("gridLayoutWidget")
self.gridLayout = QtWidgets.QGridLayout(self.gridLayoutWidget)
self.gridLayout.setContentsMargins(0, 0, 0, 0)
self.gridLayout.setObjectName("gridLayout")
# зона результатов
self.label_nbc_type = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_nbc_type.setText("")
self.label_nbc_type.setObjectName("label_nbc_type")
self.label_nbc_type.setAlignment(QtCore.Qt.AlignCenter)
self.label_nbc_type.setVisible(False)
self.gridLayout.addWidget(self.label_nbc_type, 3, 1, 1, 1)
self.label_svm_zone = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_svm_zone.setText("")
self.label_svm_zone.setObjectName("label_svm_zone")
self.label_svm_zone.setAlignment(QtCore.Qt.AlignCenter)
self.label_svm_zone.setVisible(False)
self.gridLayout.addWidget(self.label_svm_zone, 1, 2, 1, 1)
self.label_knn_zone = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_knn_zone.setText("")
self.label_knn_zone.setObjectName("label_knn_zone")
self.label_knn_zone.setAlignment(QtCore.Qt.AlignCenter)
self.label_knn_zone.setVisible(False)
self.gridLayout.addWidget(self.label_knn_zone, 2, 2, 1, 1)
self.label_nbc_zone = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_nbc_zone.setText("")
self.label_nbc_zone.setObjectName("label_nbc_zone")
self.label_nbc_zone.setAlignment(QtCore.Qt.AlignCenter)
self.label_nbc_zone.setVisible(False)
self.gridLayout.addWidget(self.label_nbc_zone, 3, 2, 1, 1)
self.label_svm_head = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_svm_head.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_svm_head.setObjectName("label_svm_head")
self.label_svm_head.setVisible(False)
self.gridLayout.addWidget(self.label_svm_head, 1, 0, 1, 1)
self.label_type_head = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_type_head.setObjectName("label_type_head")
self.label_type_head.setAlignment(QtCore.Qt.AlignCenter)
self.label_type_head.setVisible(False)
self.gridLayout.addWidget(self.label_type_head, 0, 1, 1, 1)
self.label_knn_head = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_knn_head.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_knn_head.setObjectName("label_knn_head")
self.label_knn_head.setVisible(False)
self.gridLayout.addWidget(self.label_knn_head, 2, 0, 1, 1)
self.label_zone_head = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_zone_head.setObjectName("label_zone_head")
self.label_zone_head.setAlignment(QtCore.Qt.AlignCenter)
self.label_zone_head.setVisible(False)
self.gridLayout.addWidget(self.label_zone_head, 0, 2, 1, 1)
self.label_nbc_head = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_nbc_head.setAlignment(QtCore.Qt.AlignCenter)
self.label_nbc_head.setObjectName("label_nbc_head")
self.label_nbc_head.setVisible(False)
self.gridLayout.addWidget(self.label_nbc_head, 3, 0, 1, 1)
self.label_knn_type = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_knn_type.setText("")
self.label_knn_type.setObjectName("label_knn_type")
self.label_knn_type.setAlignment(QtCore.Qt.AlignCenter)
self.label_knn_type.setVisible(False)
self.gridLayout.addWidget(self.label_knn_type, 2, 1, 1, 1)
self.label_svm_type = QtWidgets.QLabel(self.gridLayoutWidget)
self.label_svm_type.setText("")
self.label_svm_type.setObjectName("label_svm_type")
self.label_svm_type.setAlignment(QtCore.Qt.AlignCenter)
self.label_svm_type.setVisible(False)
# зона вводов
self.gridLayout.addWidget(self.label_svm_type, 1, 1, 1, 1)
self.horizontalLayoutWidget_2 = QtWidgets.QWidget(self.centralwidget)
self.horizontalLayoutWidget_2.setGeometry(QtCore.QRect(120, 110, 251, 451))
self.horizontalLayoutWidget_2.setObjectName("horizontalLayoutWidget_2")
self.horizontalLayout_4 = QtWidgets.QHBoxLayout(self.horizontalLayoutWidget_2)
self.horizontalLayout_4.setContentsMargins(0, 0, 0, 0)
self.horizontalLayout_4.setObjectName("horizontalLayout_4")
self.verticalLayout = QtWidgets.QVBoxLayout()
self.verticalLayout.setObjectName("verticalLayout")
self.label_Al = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Al.setFont(font)
self.label_Al.setTabletTracking(True)
self.label_Al.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Al.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Al.setObjectName("label_Al")
self.verticalLayout.addWidget(self.label_Al)
self.label_Ba = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Ba.setFont(font)
self.label_Ba.setTabletTracking(True)
self.label_Ba.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Ba.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Ba.setObjectName("label_Ba")
self.verticalLayout.addWidget(self.label_Ba)
self.label_Ca = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Ca.setFont(font)
self.label_Ca.setTabletTracking(True)
self.label_Ca.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Ca.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Ca.setObjectName("label_Ca")
self.verticalLayout.addWidget(self.label_Ca)
self.label_Cu = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Cu.setFont(font)
self.label_Cu.setTabletTracking(True)
self.label_Cu.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Cu.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Cu.setObjectName("label_Cu")
self.verticalLayout.addWidget(self.label_Cu)
self.label_Fe = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Fe.setFont(font)
self.label_Fe.setTabletTracking(True)
self.label_Fe.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Fe.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Fe.setObjectName("label_Fe")
self.verticalLayout.addWidget(self.label_Fe)
self.label_K = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_K.setFont(font)
self.label_K.setTabletTracking(True)
self.label_K.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_K.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_K.setObjectName("label_K")
self.verticalLayout.addWidget(self.label_K)
self.label_Li = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Li.setFont(font)
self.label_Li.setTabletTracking(True)
self.label_Li.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Li.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Li.setObjectName("label_Li")
self.verticalLayout.addWidget(self.label_Li)
self.label_Mg = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Mg.setFont(font)
self.label_Mg.setTabletTracking(True)
self.label_Mg.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Mg.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Mg.setObjectName("label_Mg")
self.verticalLayout.addWidget(self.label_Mg)
self.label_Mn = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Mn.setFont(font)
self.label_Mn.setTabletTracking(True)
self.label_Mn.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Mn.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Mn.setObjectName("label_Mn")
self.verticalLayout.addWidget(self.label_Mn)
self.label_Na = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Na.setFont(font)
self.label_Na.setTabletTracking(True)
self.label_Na.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Na.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Na.setObjectName("label_Na")
self.verticalLayout.addWidget(self.label_Na)
self.label_Ni = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Ni.setFont(font)
self.label_Ni.setTabletTracking(True)
self.label_Ni.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Ni.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Ni.setObjectName("label_Ni")
self.verticalLayout.addWidget(self.label_Ni)
self.label_Rb = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Rb.setFont(font)
self.label_Rb.setTabletTracking(True)
self.label_Rb.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Rb.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Rb.setObjectName("label_Rb")
self.verticalLayout.addWidget(self.label_Rb)
self.label_Sr = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Sr.setFont(font)
self.label_Sr.setTabletTracking(True)
self.label_Sr.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Sr.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Sr.setObjectName("label_Sr")
self.verticalLayout.addWidget(self.label_Sr)
self.label_Ti = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Ti.setFont(font)
self.label_Ti.setTabletTracking(True)
self.label_Ti.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Ti.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Ti.setObjectName("label_Ti")
self.verticalLayout.addWidget(self.label_Ti)
self.label_Zn = QtWidgets.QLabel(self.horizontalLayoutWidget_2)
font = QtGui.QFont()
font.setPointSize(10)
self.label_Zn.setFont(font)
self.label_Zn.setTabletTracking(True)
self.label_Zn.setLayoutDirection(QtCore.Qt.LeftToRight)
self.label_Zn.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter)
self.label_Zn.setObjectName("label_Zn")
self.verticalLayout.addWidget(self.label_Zn)
self.horizontalLayout_4.addLayout(self.verticalLayout)
self.verticalLayout_2 = QtWidgets.QVBoxLayout()
self.verticalLayout_2.setObjectName("verticalLayout_2")
self.lineEdit_Al = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Al.setObjectName("lineEdit_Al")
self.verticalLayout_2.addWidget(self.lineEdit_Al)
self.lineEdit_Ba = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Ba.setObjectName("lineEdit_Ba")
self.verticalLayout_2.addWidget(self.lineEdit_Ba)
self.lineEdit_Ca = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Ca.setObjectName("lineEdit_Ca")
self.verticalLayout_2.addWidget(self.lineEdit_Ca)
self.lineEdit_Cu = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Cu.setObjectName("lineEdit_Cu")
self.verticalLayout_2.addWidget(self.lineEdit_Cu)
self.lineEdit_Fe = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Fe.setObjectName("lineEdit_Fe")
self.verticalLayout_2.addWidget(self.lineEdit_Fe)
self.lineEdit_K = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_K.setObjectName("lineEdit_K")
self.verticalLayout_2.addWidget(self.lineEdit_K)
self.lineEdit_Li = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Li.setObjectName("lineEdit_Li")
self.verticalLayout_2.addWidget(self.lineEdit_Li)
self.lineEdit_Mg = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Mg.setObjectName("lineEdit_Mg")
self.verticalLayout_2.addWidget(self.lineEdit_Mg)
self.lineEdit_Mn = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Mn.setObjectName("lineEdit_Mn")
self.verticalLayout_2.addWidget(self.lineEdit_Mn)
self.lineEdit_Na = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Na.setObjectName("lineEdit_Na")
self.verticalLayout_2.addWidget(self.lineEdit_Na)
self.lineEdit_Ni = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Ni.setObjectName("lineEdit_Ni")
self.verticalLayout_2.addWidget(self.lineEdit_Ni)
self.lineEdit_Rb = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Rb.setObjectName("lineEdit_Rb")
self.verticalLayout_2.addWidget(self.lineEdit_Rb)
self.lineEdit_Sr = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Sr.setObjectName("lineEdit_Sr")
self.verticalLayout_2.addWidget(self.lineEdit_Sr)
self.lineEdit_Ti = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Ti.setObjectName("lineEdit_Ti")
self.verticalLayout_2.addWidget(self.lineEdit_Ti)
self.lineEdit_Zn = QtWidgets.QLineEdit(self.horizontalLayoutWidget_2)
self.lineEdit_Zn.setObjectName("lineEdit_Zn")
self.verticalLayout_2.addWidget(self.lineEdit_Zn)
self.horizontalLayout_4.addLayout(self.verticalLayout_2)
self.label_input_head = QtWidgets.QLabel(self.centralwidget)
self.label_input_head.setGeometry(QtCore.QRect(120, 90, 251, 16))
font = QtGui.QFont()
font.setPointSize(12)
self.label_input_head.setFont(font)
self.label_input_head.setAlignment(QtCore.Qt.AlignCenter)
self.label_input_head.setObjectName("label_input_head")
self.verticalLayoutWidget_3 = QtWidgets.QWidget(self.centralwidget)
self.verticalLayoutWidget_3.setGeometry(QtCore.QRect(119, 0, 251, 80))
self.verticalLayoutWidget_3.setObjectName("verticalLayoutWidget_3")
self.verticalLayout_3 = QtWidgets.QVBoxLayout(self.verticalLayoutWidget_3)
self.verticalLayout_3.setContentsMargins(0, 0, 0, 0)
self.verticalLayout_3.setObjectName("verticalLayout_3")
self.label_wt_head = QtWidgets.QLabel(self.verticalLayoutWidget_3)
font = QtGui.QFont()
font.setPointSize(12)
self.label_wt_head.setFont(font)
self.label_wt_head.setAlignment(QtCore.Qt.AlignCenter)
self.label_wt_head.setObjectName("label_wt_head")
self.verticalLayout_3.addWidget(self.label_wt_head)
self.horizontalLayout_2 = QtWidgets.QHBoxLayout()
self.horizontalLayout_2.setObjectName("horizontalLayout_2")
self.radioButton_white = QtWidgets.QRadioButton(self.verticalLayoutWidget_3)
self.radioButton_white.setChecked(True)
self.radioButton_white.setObjectName("radioButton_white")
self.horizontalLayout_2.addWidget(self.radioButton_white)
self.radioButton_red = QtWidgets.QRadioButton(self.verticalLayoutWidget_3)
self.radioButton_red.setLayoutDirection(QtCore.Qt.RightToLeft)
self.radioButton_red.setChecked(False)
self.radioButton_red.setObjectName("radioButton_red")
self.horizontalLayout_2.addWidget(self.radioButton_red)
self.verticalLayout_3.addLayout(self.horizontalLayout_2)
self.label_results = QtWidgets.QLabel(self.centralwidget)
self.label_results.setGeometry(QtCore.QRect(420, 180, 561, 51))
font = QtGui.QFont()
font.setPointSize(12)
self.label_results.setFont(font)
self.label_results.setAlignment(QtCore.Qt.AlignCenter)
self.label_results.setObjectName("label_results")
self.label_results.setVisible(False)
self.pushButton_calculate = QtWidgets.QPushButton(self.centralwidget)
self.pushButton_calculate.setGeometry(QtCore.QRect(420, 530, 191, 28))
self.pushButton_calculate.setObjectName("pushButton_calculate")
MainWindow.setCentralWidget(self.centralwidget)
self.statusbar = QtWidgets.QStatusBar(MainWindow)
self.statusbar.setObjectName("statusbar")
MainWindow.setStatusBar(self.statusbar)
self.pushButton_calculate.clicked.connect(self.calculate)
self.retranslateUi(MainWindow)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
def retranslateUi(self, MainWindow):
MainWindow.setWindowTitle(QtWidgets.QApplication.translate("MainWindow", "Классификатор вин", None, -1))
self.label_svm_head.setText(QtWidgets.QApplication.translate("MainWindow", "Метод опорных векторов:", None, -1))
self.label_type_head.setText(QtWidgets.QApplication.translate("MainWindow", "Наименование", None, -1))
self.label_knn_head.setText(
QtWidgets.QApplication.translate("MainWindow", "Метод k-ближайших соседей:", None, -1))
self.label_zone_head.setText(QtWidgets.QApplication.translate("MainWindow", "Зона", None, -1))
self.label_nbc_head.setText(
QtWidgets.QApplication.translate("MainWindow", "Наивный Байесовский классификатор:", None, -1))
self.label_Al.setText(QtWidgets.QApplication.translate("MainWindow", "Al", None, -1))
self.label_Ba.setText(QtWidgets.QApplication.translate("MainWindow", "Ba", None, -1))
self.label_Ca.setText(QtWidgets.QApplication.translate("MainWindow", "Ca", None, -1))
self.label_Cu.setText(QtWidgets.QApplication.translate("MainWindow", "Cu", None, -1))
self.label_Fe.setText(QtWidgets.QApplication.translate("MainWindow", "Fe", None, -1))
self.label_K.setText(QtWidgets.QApplication.translate("MainWindow", "K", None, -1))
self.label_Li.setText(QtWidgets.QApplication.translate("MainWindow", "Li", None, -1))
self.label_Mg.setText(QtWidgets.QApplication.translate("MainWindow", "Mg", None, -1))
self.label_Mn.setText(QtWidgets.QApplication.translate("MainWindow", "Mn", None, -1))
self.label_Na.setText(QtWidgets.QApplication.translate("MainWindow", "Na", None, -1))
self.label_Ni.setText(QtWidgets.QApplication.translate("MainWindow", "Ni", None, -1))
self.label_Rb.setText(QtWidgets.QApplication.translate("MainWindow", "Rb", None, -1))
self.label_Sr.setText(QtWidgets.QApplication.translate("MainWindow", "Sr", None, -1))
self.label_Ti.setText(QtWidgets.QApplication.translate("MainWindow", "Ti", None, -1))
self.label_Zn.setText(QtWidgets.QApplication.translate("MainWindow", "Zn", None, -1))
self.label_input_head.setText(QtWidgets.QApplication.translate("MainWindow", "Элементный состав", None, -1))
self.label_wt_head.setText(QtWidgets.QApplication.translate("MainWindow", "Тип вина", None, -1))
self.radioButton_white.setText(QtWidgets.QApplication.translate("MainWindow", "Белое", None, -1))
self.radioButton_red.setText(QtWidgets.QApplication.translate("MainWindow", "Красное", None, -1))
self.label_results.setText(QtWidgets.QApplication.translate("MainWindow", "Результаты", None, -1))
self.pushButton_calculate.setText(QtWidgets.QApplication.translate("MainWindow", "Классифицировать", None, -1))
@QtCore.Slot()
def calculate(self):
self.label_results.setVisible(True)
elements = ['Al', 'Ba', 'Ca', 'Cu', 'Fe', 'K', 'Li', 'Mg', 'Mn', 'Na', 'Ni', 'Rb', 'Sr', 'Ti', 'Zn']
element_values = list()
for element in elements:
element_values.append(float(convert_colons(eval('self.lineEdit_{}.text()'.format(element)))))
element_values = np.array(element_values)
if self.radioButton_white.isChecked():
wine_type = 'white'
else:
wine_type = 'red'
ret_dict = self.models.predict(wine_type, element_values)
for method in ['svm', 'knn', 'nbc']:
exec('self.label_{}_head.setVisible(True)'.format(method))
for class_type in ['type', 'zone']:
exec('self.label_{}_head.setVisible(True)'.format(class_type))
current_value = ret_dict['{}_{}'.format(method, class_type)][0]
exec('self.label_{}_{}.setText(current_value)'.format(method, class_type))
exec('self.label_{}_{}.setVisible(True)'.format(method, class_type))
def test():
models = Models()
exp = RegressionExperiment("med_random")
exp.predict(FeaturePipeline.ngram_reg(models.get("svr"), 1))
if es.ping():
print('Connected to Elasticsearch')
else:
print('Could not connect to elasticsearch')
sys.exit()
def remove_special_chars(text):
'''This function removes the special chars from the text'''
text=str(text)
text = re.sub('[^A-Za-z0-9]+', ' ', text)
text=text.lower()
return text
#Loading the Universal sentence encoder model
model=Models()
def get_query_doc(query):
query=remove_special_chars(query)
query_vector =model.get_vec_rep(query) # Getting the vector representation of text from the model
#Creating a document structure to search with query.
query_doc = {
"query" :
{
"script_score" :
{
"query" : {
"match_all": {}
},
"script" : {
"source": "cosineSimilarity(params.query_vector, 'text_vector') + 1.0",
def test_makeModels(self):
m1 = Models()
self.assertEqual(len(m1.modelList), 6)
if args.method.lower() in ['sorecgatitem']:
dataset = SocialItem_Dataset(args)
elif args.method.lower() in ['sorecgatuser']:
dataset = SocialUser_Dataset(args)
else:
dataset = Dataset(args)
params = Parameters(args, dataset)
print(
"""Load data done [%.1f s]. #user:%d, #item:%d, #dom:%d, #train:%d, #test:%d, #valid:%d"""
% (time() - t1, params.num_users, params.num_items, params.num_doms,
params.num_train_instances, params.num_test_instances,
params.num_valid_instances))
print('Method: %s' % (params.method))
if params.method in ['sorecgatitem', 'sorecgatuser']:
model = Models(params)
model.define_model()
model.define_loss('all')
print("Model definition completed: in %.2fs" % (time() - t1))
train_step = get_optimizer(params.learn_rate,
params.optimizer).minimize(model.loss)
init = tf.global_variables_initializer()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
print('train instances: {}'.format(params.train_matrix.nnz))
error_plot = Error_plot(save_flag=True,
res_path=params.result_path,
args_str=args_str,
args=args)
config_file_path = results_folder + '/params.ini'
args = train_utils.parse_params(config_file_path)
L = args.num_classes
args.image_dim = [128, 128, 3]
#%% load saved network parameters and open new session
tf.reset_default_graph()
in_placeholder = tf.placeholder(
tf.float32,
shape=[None, None, None, L + args.image_dim[2]],
name="in_placeholder")
out_placeholder = tf.placeholder(tf.float32,
shape=[None, None, None, L],
name='out_placeholder')
phase = tf.placeholder(tf.bool, name='phase')
net_class = Models(args)
net_class.build_model(in_placeholder, phase)
sess = tf.Session()
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(checkpoints_path)
if ckpt and ckpt.model_checkpoint_path:
ckpt_path = checkpoints_path + 'my_model-' + str(checkpoint[0])
saver.restore(sess, ckpt_path)
#%%
if not os.path.exists(output_path):
os.makedirs(output_path)
image_list = sorted(os.listdir(pascal_path + '/images'))
class Unshred:
def __init__(self, verbose = False):
self.encrypted_string = (
"de| | f|Cl|nf|ed|au| i|ti| |ma|ha|or|nn|ou| S|on|nd|on" "\n"
"ry| |is|th|is| b|eo|as| | |f |wh| o|ic| t|, | |he|h " "\n"
"ab| |la|pr|od|ge|ob| m|an| |s |is|el|ti|ng|il|d |ua|c " "\n"
"he| |ea|of|ho| m| t|et|ha| | t|od|ds|e |ki| c|t |ng|br" "\n"
"wo|m,|to|yo|hi|ve|u | t|ob| |pr|d |s |us| s|ul|le|ol|e " "\n"
' t|ca| t|wi| M|d |th|"A|ma|l |he| p|at|ap|it|he|ti|le|er' "\n"
'ry|d |un|Th|" |io|eo|n,|is| |bl|f |pu|Co|ic| o|he|at|mm' "\n"
"hi| | |in| | | t| | | | |ye| |ar| |s | | |. ")
self.rows = 8
self.columns = 19
self.matrix = [ [ 0 for i in range(self.columns) ] for j in range(self.rows) ]
self.combination = Combination(self.rows, self.columns)
self.used_column_indices = []
self.verbose = verbose
def iterate_lines(self, foo): return iter(foo.splitlines())
def parseIntoMatrix(self):
i = 0
for line in self.iterate_lines(self.encrypted_string):
row_values = line.split('|')
for j, value in enumerate(row_values):
if value != '':
self.matrix[i][j] = value
i += 1
def initModels(self):
self.corp = Models()
self.corp.createLetterModel()
self.corp.createWordUniGramModel()
def computeCombinationProbability(self, phrases):
sum = 0
for phrase in phrases:
sum += self.corp.getLetterProbabilities(phrase)
sum += self.corp.getWordProbabilities(phrase)
if self.verbose:
print '{0:15f} |{1:s}|'.format(sum, phrases[0])
#print sum, phrases
return sum
def getBestCombination(self):
max_prob = -100000
max_combination = ''
max_index = 0
for index, pair in enumerate(self.matrix[0]):
# Don't check columns we already added
if index in self.used_column_indices:
continue
temp_combination = copy.deepcopy(self.combination)
# Get column values for index
new_column_values = [self.matrix[i][index] for i in range(self.rows)]
temp_combination.appendColumn(new_column_values)
phrases_to_check = temp_combination.returnPhrases()
# Compute probabilities
probability = self.computeCombinationProbability(phrases_to_check)
if probability > max_prob:
max_prob = probability
max_combination = temp_combination
max_index = index
if self.verbose:
print '->{0:13f} |{1:s}|'.format(max_prob, max_combination.returnFirstPhrase())
self.used_column_indices.append(max_index)
return max_prob, max_combination
def unshred(self):
# Find uppercase
for index, pair in enumerate(self.matrix[0]):
if pair[0].isupper():
self.combination.appendColumn([self.matrix[i][index] for i in range(self.rows)])
self.used_column_indices.append(index)
break
# Find next best combinations and append them
for i in range(1, 19):
prob, combination = self.getBestCombination()
#print prob, combination
self.combination = combination
def displayResult(self):
print '\n The unencrypted message is:\n'
self.combination.display()
def execute(self):
self.parseIntoMatrix()
self.initModels()
self.unshred()
self.displayResult()
return
dataset = Dataset(args)
params = Parameters(args,dataset)
print("""Load data done [%.1f s]. #user:%d, #list:%d, #item:%d, #train:%d, #valid:%d, #test:%d"""% (time() - t1, params.num_user, params.num_list,
params.num_item,params.num_train_instances,params.num_valid_instances,params.num_test_instances))
args.args_str = params.get_args_to_string()
t1 = time()
print("args str: ",args.args_str)
print("leng from list_items_list: ",len(utils.get_value_lists_as_list(params.list_items_dct)))
print("leng from trainArrTriplets: ", len((params.trainArrTriplets[0])))
print("non-zero entries in train_matrix: ", params.train_matrix.nnz)
# model-loss-optimizer defn =======================================================================
models = Models(params,device=device)
model = models.get_model()
if params.loss not in ['bpr']: #bpr
criterion_li = torch.nn.BCELoss()
#criterion_li = torch.nn.BCEWithLogitsLoss() ## new change made
if params.optimizer == 'adam':
optimizer_gnn = torch.optim.Adam(model.parameters(), lr=params.lr)
optimizer_seq = torch.optim.Adam(model.parameters(), lr=params.lr)
elif params.optimizer == 'rmsprop':
optimizer_gnn = torch.optim.RMSprop(model.parameters(), lr=params.lr)
optimizer_seq = torch.optim.RMSprop(model.parameters(), lr=params.lr)
model.to(device)
# training =======================================================================
## param =============================
def initModels(self):
self.corp = Models()
self.corp.createLetterModel()
self.corp.createWordUniGramModel()
def Classify(X, Y, cls, rep, k=5000):
# Start moment
Start_moment = time.time()
title = 'Classificando com {} e {} k={}'.format(cls, rep, k)
print(title)
# Creating the K-fold cross validator
K_fold = KFold(n_splits=10, shuffle=True)
# Labels
test_labels = np.array([], 'int32')
test_pred = np.array([], 'int32')
# Confusion Matrix
confusion = np.array([[0, 0], [0, 0]])
# The test
for train_indices, test_indices in K_fold.split(X):
print('Running .... =)')
X_train = [X[i] for i in train_indices]
Y_train = [Y[i] for i in train_indices]
X_test = [X[i] for i in test_indices]
Y_test = [Y[i] for i in test_indices]
train_x, train_y, test_x, test_y = Representations(
).get_representation(rep=rep,
train_x=X_train,
train_y=Y_train,
test_x=X_test,
test_y=Y_test,
k=k,
cat=None)
# c = Counter(Y_train)
# print(Counter(train_y))
# print({1:c.most_common(1)[0][1], 0:c.most_common(1)[0][1], 2:c.most_common(1)[0][1]})
sm = SMOTE(sampling_strategy='minority', random_state=None)
# sm = SMOTE(sampling_strategy={1:c.most_common(1)[0][1], 0:c.most_common(1)[0][1], 2:c.most_common(1)[0][1]}, random_state=None)
# print(len(train_y))
train_x, train_y = sm.fit_sample(train_x, train_y)
# print(Counter(train_y))
test_labels = np.append(test_labels, Y_test)
classifier = Models().get_classifier(cls)
classifier.fit(train_x, train_y)
# Train_Classifier(classifier, X_train, Y_train)
pred = classifier.predict(test_x)
test_pred = np.append(test_pred, pred)
# print(test_y)
# print(pred)
confusion += confusion_matrix(test_y, pred)
# report = classification_report(test_labels, test_pred, target_names=['Contrário', 'Favorável'] if plb =='polaridade' else ['neutro', 'opiniao'])
report = classification_report(test_labels,
test_pred,
target_names=['no', 'yes'])
print(report)
print("Confusion matrix:")
print(confusion)
Finish_moment = time.time()
tm = "It took " + str((Finish_moment - Start_moment)) + " seconds"
print(tm)
def __init__(self):
# super(self).__init__()
# инициализация классификаторов
self.models = Models()
class Processor(object):
def __init__(self, seed=999, n_splits=10):
self.seed = seed
self.n_splits = n_splits
import sys
sys.path.append(
r"C:\Users\Kelvin\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation"
)
#sys.path.append(r"C:\Users\Kelvi\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation")
from Models import Models
self.models = Models()
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import TimeSeriesSplit
tscv = TimeSeriesSplit(n_splits)
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
#self.models.add_model(model = GridSearchCV(estimator=MLPClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Neural Net')
#self.models.add_model(model = GridSearchCV(estimator=KNeighborsClassifier(), param_grid={}, cv=tscv), model_name = 'KNN')
#self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='linear', random_state=seed), param_grid={}, cv=tscv), model_name = 'Linear SVM')
#self.models.add_model(model = GridSearchCV(estimator=SVC(kernel='rbf', random_state=seed), param_grid={}, cv=tscv), model_name = 'RBF SVM')
#self.models.add_model(model = GridSearchCV(estimator=GaussianProcessClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Gaussian Process')
#self.models.add_model(model = GridSearchCV(estimator=DecisionTreeClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Decision Tree')
#self.models.add_model(model = GridSearchCV(estimator=RandomForestClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'Random Forest')
#self.models.add_model(model = GridSearchCV(estimator=AdaBoostClassifier(random_state=seed), param_grid={}, cv=tscv), model_name = 'AdaBoost')
#self.models.add_model(model = GridSearchCV(estimator=GaussianNB(), param_grid={}, cv=tscv), model_name = 'Naive Bayes')
##self.models.add_model(model = GridSearchCV(estimator=QuadraticDiscriminantAnalysis(), param_grid={}, cv=tscv), model_name = 'QDA')
self.models.add_model(model=MLPClassifier(random_state=seed),
model_name='Neural Net')
self.models.add_model(model=KNeighborsClassifier(), model_name='KNN')
self.models.add_model(model=SVC(kernel='linear', random_state=seed),
model_name='Linear SVM')
self.models.add_model(model=SVC(kernel='rbf', random_state=seed),
model_name='RBF SVM')
self.models.add_model(
model=GaussianProcessClassifier(random_state=seed),
model_name='Gaussian Process')
self.models.add_model(model=DecisionTreeClassifier(random_state=seed),
model_name='Decision Tree')
self.models.add_model(model=RandomForestClassifier(random_state=seed),
model_name='Random Forest')
self.models.add_model(model=AdaBoostClassifier(random_state=seed),
model_name='AdaBoost')
self.models.add_model(model=GaussianNB(), model_name='Naive Bayes')
#self.models.add_model(model = QuadraticDiscriminantAnalysis(), model_name = 'QDA')
def set_dataset(self,
interval=30,
no_of_steps=1,
window_size=10,
interpolation_method='linear',
train_start='2009-01-01',
train_end='2009-12-31',
valid_start='2010-01-01',
valid_end='2010-12-31',
test_start='2011-01-01',
test_end='2011-12-31',
dimensions=3):
from DB import DB
from Dataset import Dataset
from Models import Models
import matplotlib.pyplot as plt
self.interval = interval
self.no_of_steps = no_of_steps
self.window_size = window_size
self.interpolation_method = interpolation_method
self.train_start = train_start
self.train_end = train_end
self.valid_start = valid_start
self.valid_end = valid_end
self.test_start = test_start
self.test_end = test_end
self.dimensions = dimensions
db = DB(driver='{SQL Server}',
server='ENVY15-NOTEBOOK\MSSQL2017',
database='DBHKUDissertation',
username='******',
password='******')
#db = DB(driver = '{SQL Server}', server = 'LAPTOP-194NACED\SQL2017', database = 'DBHKUDissertation', username = '******', password = '******')
sql = """
select
[TradingDatetime]
, [Open]
, [High]
, [Low]
, [Close]
, [Volume]
FROM [DBHKUDissertation].[dbo].[TableStock]
where Ticker in ('sh000300')
and Interval in (""" + str(interval) + """)
"""
#df = db.read_sql(sql_string = sql)
self.dataset = Dataset(data=db.read_sql(sql_string=sql),
no_of_intervals_per_day=240 / interval,
no_of_steps=no_of_steps)
#print(self.dataset.get_data().head())
#self.dataset.visualize(columns=['Close'])
print('created dataset object')
self.dataset.interpolate(method=interpolation_method)
#print(self.dataset.get_data().head())
print('interpolated')
self.dataset.derive_features(window_size)
#print(self.dataset.get_data().to_string())
print('derived features')
self.dataset.remove_na()
print(self.dataset.get_data().head())
print('removed na')
#print(self.dataset.get_data().head())
#dataset.get_data().head()['ReturnDummy'].unique()
self.dataset.data_splitting(train_start=train_start,
train_end=train_end,
valid_start=valid_start,
valid_end=valid_end,
test_start=test_start,
test_end=test_end)
#print(self.dataset.get_train().head())
print('splitted data')
self.dataset.avoid_look_ahead_bias()
#print(self.dataset.get_train().head())
print('removed look ahead')
#self.dataset.set_X(['MA' + str(window_size), 'MA' + str(window_size * 2), 'MA' + str(window_size * 3), 'MA' + str(window_size * 4), 'MA' + str(window_size * 5), 'BB' + str(window_size), 'BB' + str(window_size * 2), 'BB' + str(window_size * 3), 'RSI' + str(window_size), 'RSI' + str(window_size * 2), 'RSI' + str(window_size * 3), 'STOCHK' + str(window_size), 'STOCHK' + str(window_size * 2), 'STOCHK' + str(window_size * 3), 'STOCHKD' + str(window_size), 'STOCHKD' + str(window_size * 2), 'STOCHKD' + str(window_size * 3)])
self.dataset.set_X([
'MA' + str(window_size), 'MA' + str(window_size * 2),
'MA' + str(window_size * 3), 'BB' + str(window_size),
'BB' + str(window_size * 2), 'BB' + str(window_size * 3),
'RSI' + str(window_size), 'RSI' + str(window_size * 2),
'RSI' + str(window_size * 3), 'STOCHK' + str(window_size),
'STOCHK' + str(window_size * 2), 'STOCHK' + str(window_size * 3),
'STOCHKD' + str(window_size), 'STOCHKD' + str(window_size * 2),
'STOCHKD' + str(window_size * 3)
])
#print(self.dataset.get_train().head())
print('set X features')
self.dataset.set_y(['ReturnDummy'])
print('set Y feature')
self.dataset.set_X_train()
#print(self.dataset.get_X_train().head())
print('set X_train')
self.dataset.set_y_train()
print('set y_train')
self.dataset.set_X_valid()
print('set X_valid')
self.dataset.set_y_valid()
print('set y_valid')
self.dataset.set_X_test()
print('set X_test')
self.dataset.set_y_test()
print('set y_test')
self.dataset.normalization()
print('normalization')
self.dataset.dimension_reduction(n_components=dimensions)
print('dimension reduction')
self.dataset.print_train_test_period()
def get_dataset(self):
return self.dataset
def get_models(self):
return self.models
def train_validate_test_models(self):
from sklearn.utils import column_or_1d
from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score, precision_recall_fscore_support, classification_report
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve
plt.figure(1)
for model_name, model in self.models.get_models().items():
print(model_name)
#print(model)
model.fit(self.dataset.get_X_train(),
column_or_1d(self.dataset.get_y_train()))
y_pred = model.predict(self.dataset.get_X_test())
#print(accuracy_score(self.dataset.get_y_test(), y_pred))
print(classification_report(self.dataset.get_y_test(), y_pred))
#print(model.best_estimator_)
#print(model.best_score_)
#print(model.best_params_)
#print(model.cv_results_)
fpr, tpr, _ = roc_curve(self.dataset.get_y_test(), y_pred)
plt.plot(fpr, tpr, label=model_name)
plt.plot([0, 1], [0, 1], 'k--')
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()
def train_validate_test_model(self, model_name):
from sklearn.utils import column_or_1d
from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score, precision_recall_fscore_support, classification_report
print(model_name)
model = self.models.get_model(model_name=model_name)
model.fit(self.dataset.get_X_train(),
column_or_1d(self.dataset.get_y_train()))
y_train_pred = model.predict(self.dataset.get_X_train())
y_valid_pred = model.predict(self.dataset.get_X_valid())
y_test_pred = model.predict(self.dataset.get_X_test())
print(confusion_matrix(self.dataset.get_y_train(), y_train_pred))
print(confusion_matrix(self.dataset.get_y_valid(), y_valid_pred))
print(confusion_matrix(self.dataset.get_y_test(), y_test_pred))
train_tn, train_fp, train_fn, train_tp = confusion_matrix(
self.dataset.get_y_train(), y_train_pred).ravel()
valid_tn, valid_fp, valid_fn, valid_tp = confusion_matrix(
self.dataset.get_y_valid(), y_valid_pred).ravel()
test_tn, test_fp, test_fn, test_tp = confusion_matrix(
self.dataset.get_y_test(), y_test_pred).ravel()
return accuracy_score(
self.dataset.get_y_train(), y_train_pred
), f1_score(self.dataset.get_y_train(), y_train_pred), precision_score(
self.dataset.get_y_train(), y_train_pred), recall_score(
self.dataset.get_y_train(), y_train_pred
), train_tn, train_fp, train_fn, train_tp, accuracy_score(
self.dataset.get_y_valid(), y_valid_pred), f1_score(
self.dataset.get_y_valid(),
y_valid_pred), precision_score(self.dataset.get_y_valid(
), y_valid_pred), recall_score(
self.dataset
.get_y_valid(),
y_valid_pred
), valid_tn, valid_fp, valid_fn, valid_tp, accuracy_score(
self.dataset.get_y_test(), y_test_pred), f1_score(
self.dataset.get_y_test(),
y_test_pred), precision_score(
self.dataset.get_y_test(),
y_test_pred), recall_score(
self.dataset.get_y_test(), y_test_pred
), test_tn, test_fp, test_fn, test_tp
#def objective_function(self
# , interval = 30
# , no_of_steps = 1
# , window_size = 10
# , interpolation_method = 'linear'
# , train_start = '2009-01-01'
# , train_end = '2009-12-31'
# , test_start = '2010-01-01'
# , test_end = '2010-12-31'
# , dimensions = 3
# , model_name = 'RBF SVM'):
# self.set_dataset(interval = interval
# , no_of_steps = no_of_steps
# , window_size = window_size
# , interpolation_method = interpolation_method
# , train_start = train_start
# , train_end = train_end
# , test_start = test_start
# , test_end = test_end
# , dimensions = dimensions)
# return self.train_validate_test_model(model_name)
def exhaustive_grid_search(self,
interval=30,
min_no_of_steps=10,
max_no_of_steps=101,
no_of_steps_interval=5,
min_window_size=5,
max_window_size=100,
window_size_interval=5,
interpolation_method='linear',
train_start='2009-01-01',
train_end='2009-12-31',
valid_start='2010-01-01',
valid_end='2010-12-31',
test_start='2011-01-01',
test_end='2011-12-31',
dimensions=3,
model_name='RBF SVM'):
import numpy as np
import pandas as pd
#import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D
#from matplotlib import cm
self.model_name = model_name
#print(min_no_of_steps)
no_of_steps = np.concatenate([[1],
np.arange(min_no_of_steps,
max_no_of_steps + 1,
no_of_steps_interval)])
window_size = np.arange(min_window_size, max_window_size + 1,
window_size_interval)
#print(no_of_steps)
#print(window_size)
X, Y = np.meshgrid(no_of_steps, window_size)
#print(X)
#print(Y)
#print(type(X))
X_train_start_date = []
X_train_end_date = []
X_valid_start_date = []
X_valid_end_date = []
X_test_start_date = []
X_test_end_date = []
train_accuracy = np.zeros(X.shape)
train_F1 = np.zeros(X.shape)
train_precision = np.zeros(X.shape)
train_recall = np.zeros(X.shape)
train_true_false_ratio = np.zeros(X.shape)
train_no_of_true = np.zeros(X.shape)
train_no_of_false = np.zeros(X.shape)
train_tn = np.zeros(X.shape)
train_fp = np.zeros(X.shape)
train_fn = np.zeros(X.shape)
train_tp = np.zeros(X.shape)
valid_accuracy = np.zeros(X.shape)
valid_F1 = np.zeros(X.shape)
valid_precision = np.zeros(X.shape)
valid_recall = np.zeros(X.shape)
valid_true_false_ratio = np.zeros(X.shape)
valid_no_of_true = np.zeros(X.shape)
valid_no_of_false = np.zeros(X.shape)
valid_tn = np.zeros(X.shape)
valid_fp = np.zeros(X.shape)
valid_fn = np.zeros(X.shape)
valid_tp = np.zeros(X.shape)
test_accuracy = np.zeros(X.shape)
test_F1 = np.zeros(X.shape)
test_precision = np.zeros(X.shape)
test_recall = np.zeros(X.shape)
test_true_false_ratio = np.zeros(X.shape)
test_no_of_true = np.zeros(X.shape)
test_no_of_false = np.zeros(X.shape)
test_tn = np.zeros(X.shape)
test_fp = np.zeros(X.shape)
test_fn = np.zeros(X.shape)
test_tp = np.zeros(X.shape)
seed = np.zeros(X.shape)
n_splits = np.zeros(X.shape)
rows = 0
#print(Z)
for step_row, window_row in zip(X, Y):
columns = 0
for step, window in zip(step_row, window_row):
print(step)
print(window)
self.set_dataset(interval=interval,
no_of_steps=step,
window_size=window,
interpolation_method=interpolation_method,
train_start=train_start,
train_end=train_end,
valid_start=valid_start,
valid_end=valid_end,
test_start=test_start,
test_end=test_end,
dimensions=dimensions)
train_accuracy[rows, columns], train_F1[rows, columns], train_precision[
rows, columns], train_recall[rows, columns], train_tn[
rows, columns], train_fp[rows, columns], train_fn[
rows,
columns], train_tp[rows, columns], valid_accuracy[
rows,
columns], valid_F1[rows, columns], valid_precision[
rows,
columns], valid_recall[rows, columns], valid_tn[
rows,
columns], valid_fp[rows, columns], valid_fn[
rows,
columns], valid_tp[rows, columns], test_accuracy[
rows, columns], test_F1[
rows,
columns], test_precision[
rows,
columns], test_recall[
rows,
columns], test_tn[
rows,
columns], test_fp[
rows,
columns], test_fn[
rows,
columns], test_tp[
rows,
columns] = self.train_validate_test_model(
model_name
)
train_no_of_true[rows,
columns] = self.dataset.get_y_train_true()
train_no_of_false[rows,
columns] = self.dataset.get_y_train_false()
train_true_false_ratio[
rows, columns] = self.dataset.get_y_train_ratio()
valid_no_of_true[rows,
columns] = self.dataset.get_y_valid_true()
valid_no_of_false[rows,
columns] = self.dataset.get_y_valid_false()
valid_true_false_ratio[
rows, columns] = self.dataset.get_y_valid_ratio()
test_no_of_true[rows, columns] = self.dataset.get_y_test_true()
test_no_of_false[rows,
columns] = self.dataset.get_y_test_false()
test_true_false_ratio[
rows, columns] = self.dataset.get_y_test_ratio()
X_train_start_date.append(
self.dataset.get_X_train_start_date().strftime(
"%Y-%m-%d %H:%M:%S"))
X_train_end_date.append(
self.dataset.get_X_train_end_date().strftime(
"%Y-%m-%d %H:%M:%S"))
X_valid_start_date.append(
self.dataset.get_X_valid_start_date().strftime(
"%Y-%m-%d %H:%M:%S"))
X_valid_end_date.append(
self.dataset.get_X_valid_end_date().strftime(
"%Y-%m-%d %H:%M:%S"))
X_test_start_date.append(
self.dataset.get_X_test_start_date().strftime(
"%Y-%m-%d %H:%M:%S"))
X_test_end_date.append(
self.dataset.get_X_test_end_date().strftime(
"%Y-%m-%d %H:%M:%S"))
seed[rows, columns] = self.seed
n_splits[rows, columns] = self.n_splits
#Z[rows, columns] = self.objective_function(interval = interval
# , no_of_steps = step
# , window_size = window
# , interpolation_method = interpolation_method
# , train_start = train_start
# , train_end = train_end
# , test_start = test_start
# , test_end = test_end
# , dimensions = dimensions
# , model_name = model_name)
columns = columns + 1
rows = rows + 1
print(train_accuracy)
print(X.reshape(X.size, 1)[:, 0])
print(Y.reshape(Y.size, 1)[:, 0])
print(train_accuracy.reshape(train_accuracy.size, 1)[:, 0])
df = pd.DataFrame(data={
'interpolation_method':
interpolation_method,
'train_start':
train_start,
'train_end':
train_end,
'valid_start':
valid_start,
'valid_end':
valid_end,
'test_start':
test_start,
'test_end':
test_end,
'dimensions':
dimensions,
'model_name':
model_name,
'no_of_steps':
X.reshape(X.size, 1)[:, 0],
'window_size':
Y.reshape(Y.size, 1)[:, 0],
'train_accuracy_score':
train_accuracy.reshape(train_accuracy.size, 1)[:, 0],
'train_F1_score':
train_F1.reshape(train_F1.size, 1)[:, 0],
'train_precision_score':
train_precision.reshape(train_precision.size, 1)[:, 0],
'train_recall_score':
train_recall.reshape(train_recall.size, 1)[:, 0],
'train_roc_tn':
train_tn.reshape(train_tn.size, 1)[:, 0],
'train_roc_fp':
train_fp.reshape(train_fp.size, 1)[:, 0],
'train_roc_fn':
train_fn.reshape(train_fn.size, 1)[:, 0],
'train_roc_tp':
train_tp.reshape(train_tp.size, 1)[:, 0],
'y_train_ratio':
train_true_false_ratio.reshape(train_true_false_ratio.size, 1)[:,
0],
'y_train_true':
train_no_of_true.reshape(train_no_of_true.size, 1)[:, 0],
'y_train_false':
train_no_of_false.reshape(train_no_of_false.size, 1)[:, 0],
'valid_accuracy_score':
valid_accuracy.reshape(valid_accuracy.size, 1)[:, 0],
'valid_F1_score':
valid_F1.reshape(valid_F1.size, 1)[:, 0],
'valid_precision_score':
valid_precision.reshape(valid_precision.size, 1)[:, 0],
'valid_recall_score':
valid_recall.reshape(valid_recall.size, 1)[:, 0],
'valid_roc_tn':
valid_tn.reshape(valid_tn.size, 1)[:, 0],
'valid_roc_fp':
valid_fp.reshape(valid_fp.size, 1)[:, 0],
'valid_roc_fn':
valid_fn.reshape(valid_fn.size, 1)[:, 0],
'valid_roc_tp':
valid_tp.reshape(valid_tp.size, 1)[:, 0],
'y_valid_ratio':
valid_true_false_ratio.reshape(valid_true_false_ratio.size, 1)[:,
0],
'y_valid_true':
valid_no_of_true.reshape(valid_no_of_true.size, 1)[:, 0],
'y_valid_false':
valid_no_of_false.reshape(valid_no_of_false.size, 1)[:, 0],
'test_accuracy_score':
test_accuracy.reshape(test_accuracy.size, 1)[:, 0],
'test_F1_score':
test_F1.reshape(test_F1.size, 1)[:, 0],
'test_precision_score':
test_precision.reshape(test_precision.size, 1)[:, 0],
'test_recall_score':
test_recall.reshape(test_recall.size, 1)[:, 0],
'test_roc_tn':
test_tn.reshape(test_tn.size, 1)[:, 0],
'test_roc_fp':
test_fp.reshape(test_fp.size, 1)[:, 0],
'test_roc_fn':
test_fn.reshape(test_fn.size, 1)[:, 0],
'test_roc_tp':
test_tp.reshape(test_tp.size, 1)[:, 0],
'y_test_ratio':
test_true_false_ratio.reshape(test_true_false_ratio.size, 1)[:, 0],
'y_test_true':
test_no_of_true.reshape(test_no_of_true.size, 1)[:, 0],
'y_test_false':
test_no_of_false.reshape(test_no_of_false.size, 1)[:, 0],
'x_train_start_date':
X_train_start_date,
'x_train_end_date':
X_train_end_date,
'x_valid_start_date':
X_valid_start_date,
'x_valid_end_date':
X_valid_end_date,
'x_test_start_date':
X_test_start_date,
'x_test_end_date':
X_test_end_date,
'seed':
seed.reshape(seed.size, 1)[:, 0],
'n_splits':
n_splits.reshape(n_splits.size, 1)[:, 0],
'interval':
interval
},
index=np.arange(1, train_accuracy.size + 1))
self.X = X
self.Y = Y
self.train_accuracy = train_accuracy
self.train_F1 = train_F1
self.train_precision = train_precision
self.train_recall = train_recall
self.train_tn = train_tn
self.train_fp = train_fp
self.train_fn = train_fn
self.train_tp = train_tp
self.valid_accuracy = valid_accuracy
self.valid_F1 = valid_F1
self.valid_precision = valid_precision
self.valid_recall = valid_recall
self.valid_tn = valid_tn
self.valid_fp = valid_fp
self.valid_fn = valid_fn
self.valid_tp = valid_tp
self.test_accuracy = test_accuracy
self.test_F1 = test_F1
self.test_precision = test_precision
self.test_recall = test_recall
self.test_tn = test_tn
self.test_fp = test_fp
self.test_fn = test_fn
self.test_tp = test_tp
self.grid_search_ouput = df
return X, Y, train_accuracy, train_F1, train_precision, train_recall, train_tn, train_fp, train_fn, train_tp, valid_accuracy, valid_F1, valid_precision, valid_recall, valid_tn, valid_fp, valid_fn, valid_tp, test_accuracy, test_F1, test_precision, test_recall, test_tn, test_fp, test_fn, test_tp, df
def to_sql(self, table_name, if_exists):
from DB import DB
db = DB(driver='{SQL Server}',
server='ENVY15-NOTEBOOK\MSSQL2017',
database='DBHKUDissertation',
username='******',
password='******')
#db = DB(driver = '{SQL Server}', server = 'LAPTOP-194NACED\SQL2017', database = 'DBHKUDissertation', username = '******', password = '******')
db.to_sql(df=self.grid_search_ouput,
table_name=table_name,
if_exists=if_exists)
def delete_table(self, sql_string):
from DB import DB
db = DB(driver='{SQL Server}',
server='ENVY15-NOTEBOOK\MSSQL2017',
database='DBHKUDissertation',
username='******',
password='******')
#db = DB(driver = '{SQL Server}', server = 'LAPTOP-194NACED\SQL2017', database = 'DBHKUDissertation', username = '******', password = '******')
db.delete_table(sql_string)
def plot_grid_search(self, is_saved, save_path):
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
fig = plt.figure(figsize=(15, 9))
#fig = plt.figure()
ax = Axes3D(fig)
surf = ax.plot_surface(self.X,
self.Y,
self.valid_accuracy,
rstride=2,
cstride=2,
cmap=cm.coolwarm,
linewidth=0.5,
antialiased=True)
ax.view_init(elev=75, azim=-50)
ax.set_title('Steps vs Window Size Optimization\n, model = ' +
str(self.model_name) + ', intervale = ' +
str(self.interval) + ', dimensions = ' +
str(self.dimensions) + '\n, train_start = ' +
str(self.train_start) + ', train_end = ' +
str(self.train_end) + ', valid_start = ' +
str(self.valid_start) + ', valid_end = ' +
str(self.valid_end) + ', test_start = ' +
str(self.test_start) + ', test_end = ' +
str(self.test_end))
ax.set_xlabel('x = no_of_steps')
ax.set_ylabel('y = window_size')
ax.set_zlabel('f(x, y) = valid_accuracy_score')
fig.colorbar(surf, shrink=0.5, aspect=5)
plt.grid(True)
if is_saved == 1:
#filename = save_path + "\" + str(self.model_name) + "_" + str(self.interval) + '_' + str(self.dimensions) + '_' + str(self.train_start) + '_' + str(self.train_end) + '_' + str(self.test_start) + '_' + str(self.test_end) + '.png'
filename = save_path + '\\' + str(self.model_name) + "_" + str(
self.interval) + '_' + str(self.dimensions) + '_' + str(
self.train_start) + '_' + str(self.train_end) + '_' + str(
self.valid_start) + '_' + str(
self.valid_end) + '_' + str(
self.test_start) + '_' + str(
self.test_end) + '.png'
print(filename)
plt.savefig(fname=filename)
else:
plt.show()
def exhaustive_grid_search_models(self,
interval=30,
min_no_of_steps=5,
max_no_of_steps=301,
no_of_steps_interval=5,
min_window_size=5,
max_window_size=51,
window_size_interval=5,
interpolation_method='linear',
train_start='2009-01-01',
train_end='2009-12-31',
valid_start='2010-01-01',
valid_end='2010-12-31',
test_start='2011-01-01',
test_end='2011-12-31',
dimensions=3):
for model_name, model in self.models.get_models().items():
print(model_name)
X, Y, train_accuracy, train_F1, train_precision, train_recall, train_tn, train_fp, train_fn, train_tp, valid_accuracy, valid_F1, valid_precision, valid_recall, valid_tn, valid_fp, valid_fn, valid_tp, test_accuracy, test_F1, test_precision, test_recall, test_tn, test_fp, test_fn, test_tp, grid_search_df = self.exhaustive_grid_search(
interval=interval,
min_no_of_steps=min_no_of_steps,
max_no_of_steps=max_no_of_steps,
no_of_steps_interval=no_of_steps_interval,
min_window_size=min_window_size,
max_window_size=max_window_size,
window_size_interval=window_size_interval,
interpolation_method=interpolation_method,
train_start=train_start,
train_end=train_end,
valid_start=valid_start,
valid_end=valid_end,
test_start=test_start,
test_end=test_end,
dimensions=dimensions,
model_name=model_name)
self.to_sql(table_name='GridSearchResult', if_exists='append')
self.plot_grid_search(
is_saved=1,
save_path=
r"C:\Users\Kelvin\CloudStation\MSC COMPUTER SCIENCE\Dissertation\CODE\Dissertation\Dissertation\GridSearchImage"
)
def getModels(client):
saveTickDateInString = Properties.SAVE_TICKDATE_IN_STRING
models = Models(client, saveTickDateInString=saveTickDateInString)
return models
__author__ = 'Placinta'
from Models import Models
encrypted_string = "Esp qtcde nzyqpcpynp zy esp ezatn zq Lcetqtntlw Tyepwwtrpynp hld spwo le Olcexzfes Nzwwprp ty estd jplc".upper(
)
new_string = ''
count = len(encrypted_string)
possibilities = []
corp = Models()
corp.createWordUniGramModel()
for i in range(1, 37):
character_list = []
new_string = ""
for j in range(0, count):
if encrypted_string[j] == ' ':
character_list.append(' ')
else:
character_list.append(
chr((ord(encrypted_string[j]) - 0x41 + i) % 26 + 0x41))
new_string = "".join(character_list).lower()
possibilities.append(new_string)
#print "{0}:".format(i)
max_prob = -10000
max_string = ''
for string in possibilities:
probability = corp.getWordProbability(string)
if probability > max_prob:
max_prob = probability
max_string = string
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
test_mkdir(args.save_path)
if args.modelname == 'mnist_2nn' or args.modelname == 'mnist_cnn':
datasetname = 'mnist'
with tf.variable_scope('inputs') as scope:
inputsx = tf.placeholder(tf.float32, [None, 784])
inputsy = tf.placeholder(tf.float32, [None, 10])
elif args.modelname == 'cifar10_cnn':
datasetname = 'cifar10'
with tf.variable_scope('inputs') as scope:
inputsx = tf.placeholder(tf.float32, [None, 24, 24, 3])
inputsy = tf.placeholder(tf.float32, [None, 10])
myModel = Models(args.modelname, inputsx)
predict_label = tf.nn.softmax(myModel.outputs)
with tf.variable_scope('loss') as scope:
Cross_entropy = -tf.reduce_mean(inputsy * tf.log(predict_label), axis=1)
with tf.variable_scope('train') as scope:
optimizer = tf.train.GradientDescentOptimizer(args.learning_rate)
train = optimizer.minimize(Cross_entropy)
with tf.variable_scope('validation') as scope:
correct_prediction = tf.equal(tf.argmax(predict_label, axis=1), tf.argmax(inputsy, axis=1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
saver = tf.train.Saver(max_to_keep=3)
def execute(self):
# parameters
epsilon = .5 # exploration
epsilon_decay = 0.95
epsilon_min = 0.1
epoch = 4000 # is number of cycles...
max_memory = 2000 # NEEDS TO BE AS BIG AS AT LEAST 1 TRADING DAY!!!
batch_size = 50 # 50
sequence_length = 250 # 500
discount = 0.95
training_days = 1
testing_days = 1
features_list = list(range(1,33)) ## FULL
features_list = list(range(1,6)) ## SHORT!!
training_store = ds.DataStore(training_days=training_days, features_list=features_list, sequence_length=sequence_length)
features_length = training_store.get_features_length()
env = Trading(data_store=training_store, sequence_length=sequence_length, features_length=features_length)
num_actions = env.get_action_count() # [sell, buy, flat] # get From TRADING!!
#testing_store = ds.DataStore(training_days=training_days, testing_days=10, features_list=features_list, sequence_length=sequence_length)
mo = Models()
rms = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-06)
use_ufcnn=True
if use_ufcnn:
model = mo.model_ufcnn_concat(sequence_length=sequence_length,
features=features_length,
nb_filter=15,
filter_length=5,
output_dim=num_actions,
optimizer=rms,
loss='mse',
batch_size = batch_size,
init="normal")
base_model_name = "ufcnn"
else:
model = mo.atari_conv_model(output_dim=num_actions, features=features_length, loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size, init="normal")
base_model_name = "atari"
testing_store = ds.DataStore(training_days=training_days, testing_days=testing_days, features_list=features_list, sequence_length=sequence_length,
mean=training_store.mean, std=training_store.std)
test_env = Trading(data_store=testing_store, sequence_length=sequence_length, features_length=features_length)
#model = mo.atari_conv_model(regression=False, output_dim=num_actions, features=features_length, nb_filter=50,
# loss='mse', sequence_length=sequence_length, optimizer=rms, batch_size=batch_size)
# If you want to continue training from a previous model, just uncomment the line bellow
#mo.load_model("ufcnn_rl_training")
# Define environment/game
# Initialize experience replay object
start_time = time.time()
best_pnl = -99999.
best_rndless_pnl = -99999.
exp_replay = ExperienceReplay(max_memory=max_memory, env=env, sequence_dim=(sequence_length, features_length), discount=discount)
lineindex = 0
# Train
for e in range(epoch):
loss = 0.
game_over = False
total_reward = 0
win_cnt = 0
loss_cnt = 0
random_cnt = 0
no_random_cnt = 0
### loop over days-...
for i in range(training_days):
input_t = env.reset()
j = 0
while not game_over: # game_over ... end of trading day...
input_tm1 = input_t
#print("INPUT ",input_tm1)
# get next action
if np.random.rand() <= epsilon:
action = np.random.randint(0, num_actions, size=1)[0]
random_cnt += 1
#print("RANDOM")
else:
q = model.predict(exp_replay.resize_input(input_tm1))
action = np.argmax(q[0])
no_random_cnt += 1
#print("SELECT")
##action = np.argmax(q)
# apply action, get rewards and new state
input_t, reward, game_over, idays, lineindex = env.act(action)
if reward > 0:
win_cnt += 1
if reward < 0:
loss_cnt += 1
total_reward += reward
if reward > 1.:
reward = 1.
if reward < -1.:
reward = -1.
# store experience
exp_replay.remember([action, reward, idays, lineindex-1], game_over)
# adapt model
if j > batch_size : # do not run exp_rep if the store is empty...
inputs, targets = exp_replay.get_batch(model, batch_size=batch_size)
curr_loss = model.train_on_batch(exp_replay.resize_input(inputs), targets)
loss += curr_loss
j += 1
rndless_pnl = self.get_randomless_pnl(test_env=test_env, model=model, testing_days=testing_days)
secs = time.time() - start_time
print("Epoch {:05d}/{} | Time {:7.1f} | Loss {:11.4f} | Win trades {:5d} | Loss trades {:5d} | Total PnL {:8.2f} | Rndless PnL {:8.2f} | Eps {:.4f} | Rnd: {:5d}| No Rnd: {:5d} ".format(e, epoch, secs, loss, win_cnt, loss_cnt, total_reward, rndless_pnl, epsilon, random_cnt, no_random_cnt), flush=True)
if epsilon > epsilon_min:
epsilon *= epsilon_decay
# Save trained model weights and architecture, this will be used by the visualization code
if total_reward > best_pnl:
mo.save_model(model,base_model_name + "_rl_best")
best_pnl = total_reward
else:
mo.save_model(model,base_model_name + "_rl_training")
if rndless_pnl > best_pnl:
mo.save_model(model,base_model_name + "_rl_rndless_best")
best_rndless_pnl = rndless_pnl
testing_store = ds.DataStore(training_days=training_days,
testing_days=testing_days,
features_list=features_list,
sequence_length=sequence_length,
mean=training_store.mean,
std=training_store.std)
features_length = training_store.get_features_length()
env = Trading(data_store=testing_store,
sequence_length=sequence_length,
features_length=features_length)
num_actions = env.get_action_count(
) # [sell, buy, flat] # get From TRADING!!
mo = Models()
start_time = time.time()
best_pnl = -99999.
exp_replay = ExperienceReplay(max_memory=max_memory,
env=env,
sequence_dim=(sequence_length,
features_length))
if len(sys.argv) == 2:
model_name = sys.argv[1]
else:
model_name = None
if model_name is not None:
model = mo.load_model(model_name)
np.shape(x_train)[3] - args.num_classes)
if args.app == 'seg':
(x_test, seg_test) = (f_HDF5["test"], f_HDF5["test_gt"])
app_loss = Segmentation_Loss(args)
elif args.app == 'dehaze':
(x_test, gt_test) = (f_HDF5["test"], f_HDF5["test_gt"])
app_loss = Dehazing_Loss(args)
elif args.app == 'matte':
(x_test, x_test_1, x_test_2,
gt_test) = (f_HDF5["val"], f_HDF5["test_t2"], f_HDF5["test_t1"],
f_HDF5["test_gt"])
app_loss = Matting_Loss(args)
print("Constructing network...")
net_class = Models(args)
tf.reset_default_graph()
input_placeholder = tf.placeholder(tf.float32, (None, ) + x_train.shape[1:],
'images_with_seeds')
phase = tf.placeholder(tf.bool, name='phase')
output_from_network = net_class.build_model(input_placeholder, phase)
loss = app_loss._loss(input_placeholder, output_from_network)
samples_in_epoch = int(
np.ceil(np.shape(x_train)[0] / np.float32(args.batch_size)))
num_iter = args.num_epochs * samples_in_epoch + 1
global_step = tf.Variable(0, trainable=False)
if args.train_schedule == 'exp': # decrease learning exponentially
learning_rate = tf.train.exponential_decay(args.init_lr, global_step,samples_in_epoch*args.decay_every,\
