def __init__(self, arg):
self.arg = arg
with open(self.arg.config_path, 'r', encoding='utf-8') as f:
self.config = json.load(f)
# data_helper 定义
self.data_loader_obj = tf_idf_data_helper(stop_word_path= self.config['stop_word_path'], low_freq=self.config['low_freq'])
# all data
self.all_data, self.all_labels = self.data_loader_obj.gen_data(self.config['train_data'])
self.tf_idf_len = self.data_loader_obj.dictionary_len
self.class_nums = self.data_loader_obj.class_nums
print("<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>")
print("class nums :{}".format(self.class_nums))
print("<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>")
# train data
self.train_data_len = int(len(self.all_data)*rate)
self.train_data, self.train_labels = self.all_data[:self.train_data_len], self.all_labels[:self.train_data_len]
# eval data
self.eval_data, self.eval_labels = self.all_data[self.train_data_len:], self.all_labels[self.train_data_len:]
# 模型定义
self.model_obj = dnn(config=self.config, tf_idf_size=self.tf_idf_len, class_nums=self.class_nums)
def __init__(self, env):
self.env = env
self.model = dnn()
if (LOAD_MODEL == False):
#input_dim,output_dim,number_layers,units_per_layer
self.model.create_model(2, 2, 3, 4)
else:
self.model.load_model()
self.exploration_rate = 1
self.exploration_decrease = 0.00005
self.exploration_minimal = 0.025
self.alpha = 0.10
self.gamma = 0.99
self.pretrain_observations = 5000
self.number_episodes = 0
self.train_df = p.DataFrame(columns=[
"Observation", "Q_Values", "Action", "Reward", "new_Observation"
])
self.pretrain = False
EPOCH_NUM = 10 # number of epochs to run before saving the model
BATCH_SIZE = 256
currentEpoch = 1
print 'Parsing...'
t0 = time.time()
trainFeats, trainLabels, trainFrameNames = parse.parseTrainData(TRAIN_FEATURE_FILENAME, TRAIN_LABEL_FILENAME)
testFeats, testFrameNames = parse.parseTestData(TEST_FEATURE_FILENAME)
t1 = time.time()
print '...costs ', t1 - t0, ' seconds'
NEURON_NUM_LIST = [ len(trainFeats[0]) ] + HIDDEN_LAYER + [ labelUtil.LABEL_NUM ]
print 'Training...'
aDNN = dnn.dnn( NEURON_NUM_LIST, LEARNING_RATE, EPOCH_NUM, BATCH_SIZE, LOAD_MODEL_FILENAME )
while True:
t2 = time.time()
aDNN.train(trainFeats, trainLabels)
t3 = time.time()
print '...costs ', t3 - t2, ' seconds'
print 'Error rate: ', aDNN.errorRate
currentEpoch += EPOCH_NUM
# Saving the Neural Network Model
modelInfo = "_ER" + str(aDNN.errorRate)[2:5] \
+ "_CO" + str(aDNN.cost)[0:7] \
+ "_HL" + str(HIDDEN_LAYER[0]) + "-" + str(len(HIDDEN_LAYER)) \
def create_model(self):
model = dnn(config=self.config,
tf_idf_size=self.tf_idf_len,
class_nums=2)
return model
Y,
n_epoch=50,
validation_set=(X_test, Y_test),
show_metric=True,
batch_size=32,
shuffle=True,
run_id="nn_cat_vs_dog")
model_NN.save(os.path.join(MODEL_PATH, model_name_NN))
# predicting
predict(model_NN, 'NN_result.csv')
tf.reset_default_graph()
# training with Deep Learning Network
model_name_DNN = 'dnn_' + MODEL_NAME
model_DNN = tflearn.DNN(dnn(),
checkpoint_path='model_DNN',
max_checkpoints=10,
tensorboard_verbose=3)
model_DNN.fit(X,
Y,
n_epoch=50,
validation_set=(X_test, Y_test),
show_metric=True,
batch_size=32,
shuffle=True,
run_id="dense_model_cat_vs_dog")
model_DNN.save(os.path.join(MODEL_PATH, model_name_DNN))
# predicting
predict(model_DNN, 'DNN_result.csv')
from func import read_price, price_to_return, train_test_split
if __name__ == "__main__":
if len(sys.argv) != 2: # the program name and the datafile
# stop the program and print an error message
sys.exit("usage: eigen.py datafile ")
filename = sys.argv[1]
print("input", sys.argv[1])
price_mat = read_price(filename)
ret_mat = price_to_return(price_mat)
n, T = ret_mat.shape
x_train, y_train, x_test, y_test = train_test_split(ret_mat)
num_layer = 3 # hiddenlayer = 2; plus output layer = 3
d = 50
print("Training with num_layer:", num_layer,
", number of nodes per layer:", d)
clf = dnn(num_layer, d)
clf.fit(x_train, y_train)
y_train_hat = clf.predict(x_train)
y_train_err = y_train - y_train_hat
print("in sample avg loss:", np.mean(y_train_err**2))
y_test_hat = clf.predict(x_test)
y_test_err = y_test - y_test_hat
print("generalization avg loss:", np.mean(y_test_err**2))
def main():
segment.segmentation("data/test.txt", "data/segment_result_test.txt")
word2vec.cal_word2vec(True, "data/word2vec.model",
"data/segment_result_test.txt",
"data/data_vector_test.txt")
dnn.dnn(True, "data/dnn.model", "data/data_vector_test.txt")
np.random.shuffle(loc) X = X[loc, :] y = y[loc] #Partition into two sets / 2-fold CV parti = X.shape[0] / 10 X_train, X_test = X[:parti], X[parti:] y_train, y_test = y[:parti], y[parti:] #print np.shape(X_train), type(y_train) #print parti # In[40]: # accu1: use train as train, test as tet iter_times = 10000 dnn1 = dnn(X_train, y_train, h_size=[20, 10], niter=iter_times) import time as ti st = ti.time() dnn1.model() print ti.time() - st y_tepd = dnn1.predict(X_test[:, :]) acc1 = dnn1.accuracy(y_test[0:], y_tepd) print 'accu = ', acc1 # In[21]: # accu2: use train as test, test as train dnn2 = dnn(X_test, y_test, niter=iter_times, h_size=[20, 10]) import time as ti st = ti.time()