def main():
dataset_path = './data_set/'
# parse arguments
args = parse_args()
if args is None:
exit()
# make data set if preprocessing data not exist
if not os.listdir(dataset_path + args.type_name):
# select sequences after year 2010
prepare = prepareData(args.type_name)
ds_path = prepare.select()
# split train and test data
data = data_set(args.type_name)
data.form_dataset()
elif os.path.exists(dataset_path + args.type_name + '/data_label_' +
args.type_name + '_IVD.npz') and not os.path.exists(
dataset_path + args.type_name + '/test_' +
args.type_name + '_8:2.npz'):
print('skip preprocessing!')
data = data_set(args.type_name)
data.form_dataset()
if args.phase == 'train':
tr = train(args)
tr.run()
elif args.phase == 'test':
te = test(args)
te.run()
from data_set import data_set
from numpy import array,cosh,sinh
import math
def diff_sol(x):
sqrt3 = math.sqrt(3)
return (1 - 2*cosh(sqrt3*x)/(2*cosh(20*sqrt3)+4*sinh(sqrt3*20)/sqrt3))
X = 20
dx = (4,2,1,0.5,0.25,0.01,)
eps = (0.03,0.1,0.25,0.5,1,)
alpha = ("new","SC","DD")
print type(diff_sol)
data_list = data_set(X,dx,alpha,eps)
data_list.plot("../OutputFiles/Run11/",show = 0,fun=diff_sol,funlabel="Diff. Soln.")
import keras
import nltk
from keras.layers import Activation
from keras.utils.generic_utils import get_custom_objects
def custom_gelu(x):
return 0.5 * x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3))))
get_custom_objects().update({'custom_gelu': Activation(custom_gelu)})
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
company = '한일현대엘리베이'
data = ds.data_set(300,company,'2010.06.05','2019.05.31','2019.06.01','2019.12.31')
train_docs,test_docs = data.docs_data()
"""
tokens = [t for d in train_docs for t in d[0]]
text = nltk.Text(tokens,name='NMSC')
print(len(text.tokens))
print(len(set(text.tokens)))
selected_words = [f[0] for f in text.vocab().most_common(100)]
with open('stopword.json',"r",encoding='utf-8') as f:
selected_words = json.load(f)
"""
from lyrics import lyrics
from data_set import data_set
import pandas as pd
from nltk.tokenize import word_tokenize
ds = data_set()
ds.load_objects()
word_lists = []
names = []
for item in ds.obj_list:
names.append(item.name)
#word_lists.append(word_tokenize(item.content))
word_lists.append(item.content)
print(names)
zippedList = list(zip(names, word_lists))
df = pd.DataFrame(zippedList, columns=['song_name', 'word_list'])
print(df.shape)
with open('all_songs_cleaned.csv', 'w', encoding="utf-8") as f:
f.write(df.to_csv(index=True))
import pandas as pd
import numpy as np
import glob
import tensorflow.compat.v1 as tensorflow
# import tensorflow
tensorflow.disable_v2_behavior()
tensorflow.compat.v1.enable_eager_execution()
import tensorflow_addons as tfa
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import concatenate, Reshape, TimeDistributed, concatenate, Dense, BatchNormalization, Dropout, Input, Flatten, Activation, Conv1D, Conv2D, LSTM, AveragePooling1D, AveragePooling2D
from data_set import data_set
print('Loading --> ')
data_set = data_set(load_n=-1,
shuffle=True,
return_joined=True,
n_samples=8192)
x_train, y_train, x_test, y_test = data_set.generate()
cat_features = data_set.get_cat_features()
n_labels = data_set.get_n_classes()
print('number of cat features: ', cat_features)
print('number of classes: ', n_labels)
def load_leads_model(trainable, drop, interdrop):
print('loading joined model')
loaded_model = tensorflow.keras.models.load_model('./deep-encoder.h5',
compile=False)
model = Sequential()
done = False
for l in loaded_model.layers:
print('Iteration %d: train_loss = %.4f[%.1f s], eval_loss = %.4f[%.1f s], NDCG@%d = %.4f'% (epoch+1, np.mean(losses), train_time, np.mean(eval_loss), eval_time, parameters.top_k, ndcgs[parameters.top_k]))
displayResult("Precision", precisions)
displayResult("Recall", recalls)
displayResult("F1", F1s)
displayResult("NDCG" ,ndcgs)
displayResult("1-call", one_calls)
print()
if ndcgs[parameters.top_k] > best_ndcg_5: # evaluation() has insured that the real index starts from 1, eg: ndcgs[5] is the fifth ndcg value(ndcg@5)
best_prec, best_rec, best_f1, best_ndcg, best_1call, best_iter = precisions, recalls, F1s, ndcgs, one_calls, epoch+1
best_ndcg_5 = ndcgs[parameters.top_k]
patience_count = 0
else:
patience_count += 1
if patience_count > parameters.patience:
break
print("End. Best Iteration %d: NDCG@%d = %.4f " % (best_iter, parameters.top_k, best_ndcg_5))
displayResult("Precision", best_prec)
displayResult("Recall", best_rec)
displayResult("F1", best_f1)
displayResult("NDCG", best_ndcg)
displayResult("1-call", best_1call)
if __name__ == '__main__':
parameters = parameters()
MLP = MLP(parameters)
data_set = data_set(parameters.train_data, parameters.test_data)
main(MLP, data_set, parameters)
from data_set import data_set
from numpy import array,cosh,sinh
import math
def diff_sol(x):
return 3.0 * ( 400 - x * x ) / 2 + 40
X = 20
eps = (0.03,0.1,0.25,0.5,1,)
dx = (4,2,1,0.5,0.25,0.01,)
alpha = ("new","SC")
data_list = data_set(X,dx,alpha,eps,path_base="../OutputFiles/Run13/output_soln_13_",dx_eps_scale = 0)
data_list.plot("../OutputFiles/Run13/",show = 0,fun=diff_sol,funlabel="Diff. Soln.")
with open("./factor_solve_data.pkl", 'rb') as f:
# pkl_file_read = read_file("factor_solve_data.pkl")
# factor_data = pickle.loads(StringIO(pkl_file_read))
factor_data = pickle.load(f, encoding='iso-8859-1')
print(factor_data.keys())
#
#print(factor_data['2009-12-31'].head())
peroid='M'
start_date='2010-01-01'
end_date='2018-01-01'
industry_old_code=['801010','801020','801030','801040','801050','801080','801110','801120','801130','801140','801150',\
'801160','801170','801180','801200','801210','801230']
industry_new_code=['801010','801020','801030','801040','801050','801080','801110','801120','801130','801140','801150',\
'801160','801170','801180','801200','801210','801230','801710','801720','801730','801740','801750',\
'801760','801770','801780','801790','801880','801890']
dateList=list(factor_data.keys())
train_set = pd.DataFrame()
test_set = dict()
train_set = data_set(dateList,factor_data,4,train_set,True)
test_set = data_set(dateList,factor_data,4,test_set,False)
to_csv("train_set.csv",train_set)
save = pickle.dumps(test_set)
with open("test_set.pkl",'wb+') as f:
f.write(save)
from data_set import data_set
from numpy import array,cosh,sinh
import math
def diff_sol(x):
sqrt3 = math.sqrt(3)
return (1 - 2*cosh(sqrt3*x)/(2*cosh(20*sqrt3)+4*sinh(sqrt3*20)/sqrt3))
X = 20
eps = (0.03,0.1,0.25,0.5,1,)
dx = (4,2,1,0.5,0.25,0.01,)
alpha = ("new","SC","DD")
data_list = data_set(X,dx,alpha,eps,path_base="../OutputFiles/Run12/output_12_",dx_eps_scale = 1)
data_list.plot("../OutputFiles/Run12/",show = 0,fun=diff_sol,funlabel="Diff. Soln.")
import obd
import data_set
import sys
import string
import time
elm327 = obd.OBD()
data = data_set.data_set()
last_torque = 0
last_power = 0
def parse_input(input):
parts = input.split(":")
try:
# parse the user's input
command = None
if len(input) == 2:
command = obd.commands[int(input, 16)]
elif len(parts) == 2:
mode = int(parts[0], 16)
pid = int(parts[1], 16)
command = obd.commands[mode][pid]
# send command and print result
if command is not None and elm327.has_command(command):
r = elm327.query(command)