def fit(self, X, y, plot_cost=False):
X_train, Y_train, X_test, Y_test = get_train_test(X,
y,
percent_train=0.7)
n, d = X_train.shape
k = Y_train.shape[1]
self.W1, self.b1 = init_weight_bias(d, self.hidden_layer_sizes[0])
self.W2, self.b2 = init_weight_bias(self.hidden_layer_sizes[0], k)
costs = []
best_validation_error = 1
if (self.batch_size == 'auto'):
self.batch_size = min(200, n)
num_batches = int(n / self.batch_size)
for i in range(self.max_iter):
X_temp, Y_temp = shuffle(X_train, Y_train)
for j in range(num_batches):
X_temp, Y_temp = X_train[
j * self.batch_size:j * self.batch_size +
self.batch_size, :], Y_train[j * self.batch_size:j *
self.batch_size +
self.batch_size, :]
Ypred, Z1 = self.forward(X_temp)
pY_t = Ypred - Y_temp
self.W2 -= self.learning_rate_init * (Z1.T.dot(pY_t))
self.b2 -= self.learning_rate_init * (pY_t.sum(axis=0))
dZ = pY_t.dot(self.W2.T) * (Z1 > 0)
self.W1 -= self.learning_rate_init * X_temp.T.dot(dZ)
self.b1 -= self.learning_rate_init * dZ.sum(axis=0)
if (i % 2) == 0:
pY_test, _ = self.forward(X_test)
c = cost(Y_test, pY_test)
costs.append(c)
e = error_rate(Y_test.argmax(axis=1), pY_test.argmax(axis=1))
print('Iteration', i, 'Cost:', c, 'Error Rate:', e)
if e < best_validation_error:
best_validation_error = e
print("best_validation_error:", best_validation_error)
if plot_cost:
plt.plot(costs)
plt.show()
def industry_embedding_model_fit(model, embedding_model, nb_epoch, batch_size,
df, dict_sequence, industry2idx, output_file):
X_train, X_test, y_train, y_test, y_train_2digit, y_test_2digit = utils.get_train_test(
df, batch_size, 0.1)
target_grid_col = 'naics_4_digit'
context_grid_col = 'context'
target_col = 'target_desc'
context_col = 'context_desc'
model.fit(x=[
X_train[target_grid_col], X_train[context_grid_col],
utils.rebuild_array(X_train[target_col]),
utils.rebuild_array(X_train[context_col])
],
y=[y_train, y_train_2digit],
epochs=nb_epoch,
shuffle=True,
batch_size=batch_size,
verbose=1,
validation_data=([
X_test[target_grid_col], X_test[context_grid_col],
utils.rebuild_array(X_test[target_col]),
utils.rebuild_array(X_test[context_col])
], [y_test, y_test_2digit]),
callbacks=[
EarlyStopping(monitor='val_loss',
min_delta=0.0001,
patience=3,
verbose=1,
mode='auto')
])
sequence_df = pd.DataFrame(
{k: v
for k, v in dict_sequence.items() if len(k) == SELECT_DIGIT}).T
naics_idx_l = np.array([
industry2idx.get(int(naics),
len(industry2idx) + 1) for naics in sequence_df.index
])
in_data = [naics_idx_l, sequence_df.values]
embedding_vec = embedding_model.predict(x=in_data, verbose=1, batch_size=1)
utils.save_weights(output_file, embedding_vec, naics_idx_l, idx2industry)
return embedding_vec
import tensorflow as tf
num_vars = 4
# (seq_length, self.n_lat, self.n_lon, self.NUM_INPUT_VARS),
seq_length = 24
epochs = 100
batch_size = 20
#Xtrain_dummy = tf.ones((batch_size, seq_length, 81, 161, num_vars))
#ytrain_dummy = tf.ones((batch_size, seq_length, 81, 161))
# antall filrer i hver lag.
filters = [256, 256, 256]
# size of filters used
kernels = [5, 5, 5]
from utils import get_xarray_dataset_for_period, get_data_keras, get_train_test
#data = get_xarray_dataset_for_period(start = '2012-01-01', stop = '2012-01-31')
#print(data)
test_start = '2014-01-01'
test_stop = '2018-12-31'
train_dataset, test_dataset = get_train_test(test_start, test_stop, model = 'ar')
X_train, y_train = get_data_keras(train_dataset, num_samples = None, seq_length = 24,
batch_size = None, data_format='channels_last')
model = ConvLSTM(X_train=X_train, y_train=y_train, filters=filters,
kernels=kernels, seq_length = seq_length,
epochs=epochs, batch_size = batch_size, validation_split=0.1,
name = 'Model5', result_path = '/home/hannasv/results/')
from utils import get_train_test
from sklearn.externals import joblib
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.multioutput import MultiOutputRegressor
from sklearn.svm import SVR
import warnings
import matplotlib.pyplot as plt
import pickle
warnings.filterwarnings('ignore')
model = MultiOutputRegressor(LinearRegression(), n_jobs=-1)
x_train, y_train, x_test, y_test, high_scale, low_scale = get_train_test(
) # train test split of dataset
# model = LinearRegression()
model.fit(x_train, y_train)
model.predict()
with open('model_scalars.pkl', 'wb') as f: # Python 3: open(..., 'wb')
pickle.dump([high_scale, low_scale, model], f)
test = model.predict(x_test)
y_trueh = high_scale.inverse_transform(y_test[:, 0].reshape(-1, 1)).ravel(
) # inverse scaling of the original test data. inverse scaling is done to get the original stock value which was converted into 0 and 1 range
y_predh = high_scale.inverse_transform(test[:, 0].reshape(
-1, 1)).ravel() # inverse scaling of the predicted data
y_truel = low_scale.inverse_transform(y_test[:, 1].reshape(-1, 1)).ravel(
) # inverse scaling of the original test data. inverse scaling is done to get the original stock value which was converted into 0 and 1 range
# data = pd.read_csv('./data.csv')
# # Y_data = pd.read_csv('./taxi_01.csv', usecols=Y_cols)
# print(data.columns)
# train, test = train_test_split(data, test_size=0.2)
# Y_train = train['total_amount']
# X_train = train.drop(['total_amount'], axis=1)
# Y_test = test['total_amount']
# X_test = test.drop(['total_amount'], axis=1)
# print(X_train.head())
# X_train = normalization(X_train)
# X_test = normalization(X_test)
X_train, Y_train, X_test, Y_test = utils.get_train_test()
# create dataset for lightgbm
lgb_train = lgb.Dataset(X_train, Y_train)
lgb_eval = lgb.Dataset(X_test, Y_test)
# specify your configurations as a dict
params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': {'l2', 'l1'},
'num_leaves': 100,
'max_depth': 15,
'learning_rate': 0.05,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from sklearn.metrics import mean_squared_error
from matplotlib import pyplot
from utils import get_train_test
from sklearn.externals import joblib
import numpy as np
import pandas as pd
x_train, y_train, x_test, y_test, scale = get_train_test(
) # train test split of dataset
# joblib.dump(scale, 'scale_models/scale_google.pkl')
x_train = np.reshape(
x_train,
(x_train.shape[0], 1, x_train.shape[1])) # reshaping data for LSTM
x_test = np.reshape(
x_test, (x_test.shape[0], 1, x_test.shape[1])) # reshaping data for LSTM
model = Sequential()
model.add(LSTM(4, input_shape=(1, 1)))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
# callback = ModelCheckpoint("weights/google.hdf5", monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=True, mode='min', period=1)
# model.fit(x_train, y_train, shuffle=False,epochs=100, batch_size=1, verbose=2, callbacks=[callback], validation_data=(x_test,y_test)) #shuffle should be set to False in time series prediction problem because sequence of stock prices matters .
model.load_weights('weights/google.hdf5')
test = model.predict(x_test)
y_true = scale.inverse_transform(y_test).ravel(
from genre_model import train, get_model
from utils import get_train_test
from genre_prediction import predict
print('Music Genre Detection menu \n')
exit = 0
X_train, Y_train, x_test, y_test = get_train_test()
print(x_test.shape)
while (1):
choice = int(
input(
'What would you like to do?\n 1.Train the network\n 2.Run Predictions on new songs\n 3.Exit\n'
))
if (choice == 1):
print('Training the model...\n')
model = get_model()
train(X_train, Y_train, x_test, y_test, model)
elif (choice == 2):
print('Making predictions...\n')
predict(x_test)
elif (choice == 3):
break
sys.path.insert(0, config.path_to_code)
# train the models for each target
for idx in range(len(type_loss)):
nb_model = 1 # begin with the first model of the ensemble
list_models = []
list_docs_val = []
# train each model of the ensemble
for i in range(len(type_loss)):
# get the docs and objective of the corresponding target
name_doc = list_name_documents[i]
docs_train, target_train, docs_val, target_val = get_train_test(
name_doc=name_doc, idx_target=idx, config=config)
list_docs_val.append(docs_val)
# take the type of loss chosen for the model
if type_loss[i] == "mse":
custom_loss = "mean_squared_error"
elif type_loss[i] == "higher":
custom_loss = mse_asymetric_higher
elif type_loss[i] == "lower":
custom_loss = mse_asymetric_lower
# take (create) the type of model chosen
if type_model[i] == 1:
model = get_model_1(docs_train=docs_train,
config=config,
name_embeddings=list_embeddings[i],
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 8 17:11:41 2018
@author: xujq
"""
import pandas as pd
''' load data '''
from utils import splitXy, get_train_test
sample = pd.read_csv("data/sample.csv")
train = pd.read_csv("data/train.csv")
X, y = splitXy(train)
X_train, X_test, y_train, y_test = get_train_test(X, y)
''' load model '''
from keras.models import load_model
model = load_model("model/my_model(3-8 16:44).h5")
warnings.filterwarnings("ignore")
TEST = True
SAVE = True
LOWE = False
descriptor = "SIFT"
descriptor = "spSIFT"
if TEST:
prefix = "%s_%s_" % (descriptor, "test")
else:
prefix = "%s_%s_" % (descriptor, "full")
train_images, train_labels, test_images, test_labels = get_train_test(TEST)
if descriptor == "SIFT":
if os.path.isfile(prefix % "kmeans"):
kmeans = load_pickle(prefix + "kmeans.pkl")
else:
pool = Pool(getNumberOfCPUs() - 2)
if LOWE:
print " [!] Lowe's SIFT"
train_sift_with_null = pool.map(get_sift_lowe, train_images)
test_sift_with_null = pool.map(get_sift_lowe, test_images)
else:
print " [!] OpenCV2's SIFT"
train_sift_with_null = pool.map(get_sift, train_images)
test_sift_with_null = pool.map(get_sift, test_images)
import numpy as np
import utils as U
Dataset_add = './DIBCO/'
patch_h = 128
patch_w = 128
N_patches = 100
(Images, Masks), (Te_d, Te_m ) = U.get_train_test(add = Dataset_add, Test_year = 2016)
patches_image, patches_masks = U.extract_random(Images, Masks, patch_h, patch_w, N_patches)
np.save('patches_image', patches_image)
np.save('patches_masks', patches_masks)
print('Done')
import warnings
warnings.filterwarnings("ignore")
TEST = True
SAVE = True
LOWE = False
descriptor = "SIFT"
descriptor = "spSIFT"
if TEST:
prefix = "%s_%s_" % (descriptor, "test")
else:
prefix = "%s_%s_" % (descriptor, "full")
train_images, train_labels, test_images, test_labels = get_train_test(TEST)
if descriptor == "SIFT":
if (os.path.isfile(prefix%"kmeans")):
kmeans = load_pickle(prefix + "kmeans.pkl")
else:
pool = Pool(getNumberOfCPUs()-2)
if LOWE:
print " [!] Lowe's SIFT"
train_sift_with_null = pool.map(get_sift_lowe, train_images)
test_sift_with_null = pool.map(get_sift_lowe, test_images)
else:
print " [!] OpenCV2's SIFT"
train_sift_with_null = pool.map(get_sift, train_images)
test_sift_with_null = pool.map(get_sift, test_images)
def main():
print(tf.__version__)
arg = get_args()
print("arg.is_valid", arg.is_valid, "type(arg.is_valid)",
type(arg.is_valid))
used_labels = None
if arg.dataset == "IN":
X = scio.loadmat(
"data/Indian_pines_corrected.mat")["indian_pines_corrected"]
y = scio.loadmat("data/Indian_pines_gt.mat")["indian_pines_gt"]
VAL_SIZE = 1025
used_labels = [1, 2, 4, 7, 9, 10, 11, 13]
elif arg.dataset == "PU":
X = scio.loadmat("data/PaviaU.mat")["paviaU"]
y = scio.loadmat("data/PaviaU_gt.mat")["paviaU_gt"]
VAL_SIZE = 4281
elif arg.dataset == "KSC":
X = scio.loadmat("data/KSC.mat")["KSC"]
y = scio.loadmat("data/KSC_gt.mat")["KSC_gt"]
elif arg.dataset == "Salinas":
X = scio.loadmat("data/Salinas_corrected.mat")["salinas_corrected"]
y = scio.loadmat("data/Salinas_gt.mat")["salinas_gt"]
elif arg.dataset == "Houston":
X = scio.loadmat("data/houston15.mat")['data']
mask_train = scio.loadmat(
"data/houston15_mask_train.mat")["mask_train"]
mask_test = scio.loadmat("data/houston15_mask_test.mat")["mask_test"]
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8)
#X_train, y_train = oversampleWeakClasses(X_train, y_train)
X = standartizeData(X)
#X_train, y_train, X_test, y_test = build_data_v2(X, y)
margin = 6
X = padWithZeros(X, margin=margin)
if arg.dataset == "Houston":
num_classes = 15
elif used_labels is not None:
num_classes = len(used_labels)
else:
num_classes = len(np.unique(y)) - 1
xshape = X.shape[1:]
aoa = []
aaa = []
ak = []
aapc = []
for repterm in range(arg.repeat_term):
if arg.dataset != "Houston":
coords, labels = get_coordinates_labels(y)
# train_coords, test_coords, train_labels, test_labels = train_test_split(coords, labels, test_size=arg.test_size)
train_coords, train_labels, test_coords, test_labels = get_train_test(
data=coords,
data_labels=labels,
test_size=arg.test_size,
limited_num=arg.limited_num,
used_labels=used_labels)
else:
train_coords, train_labels = get_coordinates_labels(mask_train)
test_coords, test_labels = get_coordinates_labels(mask_test)
train_coords = train_coords + margin
test_coords = test_coords + margin
#X_test = Grammar(X, test_coords, method=arg.test_region)
X_train = Grammar(X, train_coords, method="rect 11")
y_train = train_labels
y_test = test_labels
if arg.data_augment:
X_train, y_train = zmm_random_flip(
X_train, y_train) # rotation_and_flip(X_train, y_train)
# X_train, y_train, X_test, y_test = build_data(X, y)
X_train_shape = X_train.shape
#X_test_shape = X_test.shape
if len(X_train_shape) == 4:
X_train = np.reshape(X_train, [
X_train_shape[0], X_train_shape[1] * X_train_shape[2],
X_train_shape[3]
])
#X_test = np.reshape(X_test, [X_test_shape[0], X_test_shape[1] * X_test_shape[2], X_test_shape[3]])
#X_test = zeropad_to_max_len(X_test, max_len=arg.max_len)
X_train = zeropad_to_max_len(X_train, max_len=arg.max_len)
for i in range(num_classes):
print("num train and test in class %d is %d / %d" %
(i, (y_train == i).sum(), (y_test == i).sum()))
#print("num_train", X_train.shape[0])
#print("num_test", X_test.shape[0])
print("num_classes", num_classes)
train_generator = Data_Generator(X,
y=y_train,
use_coords=train_coords,
batch_size=arg.batch_size,
selection_rules=selection_rules,
shuffle=True,
till_end=False,
max_len=arg.max_len)
test_generator = Data_Generator(X,
y=y_test,
use_coords=test_coords,
batch_size=1024,
selection_rules=[arg.test_region],
shuffle=False,
till_end=True,
max_len=arg.max_len)
model = HSI_BERT(max_len=arg.max_len,
n_channel=xshape[-1],
max_depth=arg.max_depth,
num_head=arg.num_head,
num_hidden=arg.num_hidden,
drop_rate=arg.drop_rate,
attention_dropout=arg.attention_dropout,
num_classes=num_classes,
start_learning_rate=arg.start_learning_rate,
prembed=arg.prembed,
prembed_dim=arg.prembed_dim,
masking=arg.masking,
pooling=arg.pooling,
pool_size=arg.pool_size)
model.build()
print(arg)
save_full_path = None
if arg.save_model:
if not os.path.exists(arg.save_path):
os.mkdir(arg.save_path)
save_full_path = arg.save_path + '/' + arg.dataset + "/model_%d_h%d_d%d" % (
repterm, arg.num_head,
arg.max_depth) + '/' + "model_%d_h%d_d%d.ckpt" % (
repterm, arg.num_head, arg.max_depth)
model_path = arg.save_path + '/' + arg.dataset + "/model_%d_h%d_d%d" % (
repterm, arg.num_head, arg.max_depth)
if not os.path.exists(model_path):
os.mkdir(model_path)
np.save(os.path.join(model_path, "train_coords.npy"),
train_coords - margin)
np.save(os.path.join(model_path, "test_coords.npy"),
test_coords - margin)
#if arg.dataset == "Salinas":
"""
print("Fitting generator")
with timer("Fitting Generator Completed"):
model.fit_generator(train_generator,
nb_epochs = arg.n_epochs,
log_every_n_samples = arg.log_every_n_samples,
save_path=save_full_path)
#preds = model.predict_from_generator(test_generator)
"""
print("Fitting normal data")
with timer("Fitting Normal Data Completed"):
model.fit(X_train,
y_train,
batch_size=arg.batch_size,
nb_epochs=arg.n_epochs,
log_every_n_samples=arg.log_every_n_samples,
save_path=save_full_path)
with timer("Testing"):
preds = model.predict_from_generator(test_generator)
result = get_matrics(y_true=test_labels, y_pred=preds)
oa = result['oa']
aa = result["aa"]
kappa = result["k"]
apc = result["apc"]
print("oa", oa)
print('aa', aa)
print("kappa", kappa)
print("apc", apc.flatten())
best_model = HSI_BERT(max_len=arg.max_len,
n_channel=xshape[-1],
max_depth=arg.max_depth,
num_head=arg.num_head,
num_hidden=arg.num_hidden,
drop_rate=arg.drop_rate,
attention_dropout=arg.attention_dropout,
num_classes=num_classes,
start_learning_rate=arg.start_learning_rate,
prembed=arg.prembed,
prembed_dim=arg.prembed_dim,
masking=arg.masking,
pooling=arg.pooling,
pool_size=arg.pool_size)
best_model.restore(save_full_path)
#if arg.dataset == "Salinas":
# preds = best_model.predict_from_generator(test_generator)
#else:
preds = best_model.predict_from_generator(test_generator)
result = get_matrics(y_pred=preds, y_true=test_labels)
oa = result['oa']
aa = result["aa"]
kappa = result["k"]
apc = result["apc"]
print("oa", oa)
print('aa', aa)
print("kappa", kappa)
print("apc", apc.flatten())
aoa.append(oa)
aaa.append(aa)
ak.append(kappa)
aapc.append(apc)
print(classification_report(test_labels, preds))
aoa = np.array(aoa)
aaa = np.array(aaa)
ak = np.array(ak)
std_aa = np.std(aaa)
std_oa = np.std(aoa)
std_ak = np.std(ak)
aapc = np.concatenate(aapc, axis=0)
print("mean oa", np.mean(aoa))
print("std_oa", std_oa)
print("mean aa", np.mean(aaa))
print("std_aa", std_aa)
print("mean kappa", np.mean(ak))
print("sta_ak", std_ak)
print("maapc", np.mean(aapc, axis=0))
print("maapc_std", np.std(aapc, axis=0))
print("below is aapc")
print(aapc)
print("===============================")
if len(sys.argv) < 3:
print("USAGE: %s <dataset> <sessions_max>" % sys.argv[0])
print(
"\tdataset - the path to the dataset from Yandex Relevance Prediction Challenge"
)
print(
"\tsessions_max - the maximum number of one-query search sessions to consider"
)
print("")
sys.exit(1)
search_sessions_path = sys.argv[1]
search_sessions_num = int(sys.argv[2])
train_sessions, train_queries, test_sessions, test_queries = utils.get_train_test(
search_sessions_path, search_sessions_num)
print("-------------------------------")
print("Training on %d search sessions (%d unique queries)." %
(len(train_sessions), len(train_queries)))
print("-------------------------------")
print("-------------------------------")
print("Testing on %d search sessions (%d unique queries)." %
(len(test_sessions), len(test_queries)))
print("-------------------------------")
loglikelihood = LogLikelihood()
perplexity = Perplexity()
for click_model in CLICK_MODELS:
