from utility import generate_training_data
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
print("Generating Training Datasets")
training_data_x, training_data_y = generate_training_data()
print("Datasets Generated")
model = Sequential()
model.add(Dense(units=9, input_dim=7))
model.add(Dense(units=15, activation='relu'))
model.add(Dense(output_dim=3, activation='softmax'))
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['accuracy'])
model.fit((np.array(training_data_x).reshape(-1, 7)),
(np.array(training_data_y).reshape(-1, 3)),
batch_size=256,
epochs=3)
model.save_weights('dnn_model.h5')
model_json = model.to_json()
with open('model.json', 'w') as json_file:
json_file.write(model_json)
file_list = []
file_list = os.listdir(path)
#file_list = ["subset.txt"]
window_size = 1
for f in file_list:
print("\n K fold Linear Regression with best 6 features on file : ", f)
print("\n window size : ", window_size)
f = path + f
training_data = utility.generate_training_data(f, 1, False, 0, 0, True)
training_data.pop(0)
#print training_data
X = utility.get_feature_matrix(training_data, window_size)
#print "matrix"
#print X
# numpy.roll( ) is used for circular shifting of elements
train_label = X[2:, 0]
train_features = X[1:- 1, [0,1,2,6,10,11]]
model = linear_model.LinearRegression()
#model = neural_network.MLPRegressor([10, 40, 5], 'relu', 'adam', 0.0001, 200, 'constant', 0.001, 0.5, 200,
# True, None, 0.0001, False, False, 0.9, True, False, 0.1, 0.9, 0.999, 1e-08)
test_files = [['sample1_period1', 'sample4_period1', 'sample5_period1', 'sample8_period1'],
['sample2_period2', 'sample6_period2', 'sample9_period2'],
['sample3_period3', 'sample7_period3', 'sample10_period3']]
for f in file_list:
print("\nHashtag File ", f)
f = path + f
training_data = []
for i in range(3):
if i == 0:
print "Regression on data before Feb 1 8 am"
training_data = utility.generate_training_data(f, 1, True, 0, epoch_8am, True)
elif i == 1:
print "Regression on data between Feb 8 am to 8 pm"
training_data = utility.generate_training_data(f, 1, True, epoch_8am, epoch_8pm, True)
else:
print "Regression on data after Feb 8 pm"
training_data = utility.generate_training_data(f, 1, True, epoch_8pm, 0, True)
if len(training_data) != 0:
training_data.pop(0)
X = numpy.matrix(training_data)
rows = X.shape[0]
avg_error = 0.0
hashtag_rmse = 0.0
#generate data for window & perform 10-fold cross-validation and regression
#path = "F:/tweets/"
file_list = []
file_list = os.listdir(path)
#file_list = ["subset.txt"]
window_size = 1
for f in file_list:
print("\n K fold Linear Regression with best 6 features on file : ", f)
print("\n window size : ", window_size)
f = path + f
training_data = utility.generate_training_data(f, 1, False, 0, 0, True)
training_data.pop(0)
#print training_data
X = utility.get_feature_matrix(training_data, window_size)
#print "matrix"
#print X
# numpy.roll( ) is used for circular shifting of elements
train_label = X[2:, 0]
train_features = X[1:-1, [0, 1, 2, 6, 10, 11]]
model = linear_model.LinearRegression()
#model = neural_network.MLPRegressor([10, 40, 5], 'relu', 'adam', 0.0001, 200, 'constant', 0.001, 0.5, 200,
# True, None, 0.0001, False, False, 0.9, True, False, 0.1, 0.9, 0.999, 1e-08)
'sample1_period1', 'sample4_period1', 'sample5_period1', 'sample8_period1'
], ['sample2_period2', 'sample6_period2', 'sample9_period2'],
['sample3_period3', 'sample7_period3', 'sample10_period3']]
for f in file_list:
print("\nHashtag File ", f)
f = path + f
training_data = []
for i in range(3):
if i == 0:
print "Regression on data before Feb 1 8 am"
training_data = utility.generate_training_data(
f, 1, True, 0, epoch_8am, True)
elif i == 1:
print "Regression on data between Feb 8 am to 8 pm"
training_data = utility.generate_training_data(
f, 1, True, epoch_8am, epoch_8pm, True)
else:
print "Regression on data after Feb 8 pm"
training_data = utility.generate_training_data(
f, 1, True, epoch_8pm, 0, True)
if len(training_data) != 0:
training_data.pop(0)
X = numpy.matrix(training_data)
rows = X.shape[0]
avg_error = 0.0