def __init__(self, epochs=None, batch_size=None, timestep=None, save=True):
self.save = save
self.myModelHandler = ModelHandler()
# Hyperparameters: User chosen parameters
self._epochs = epochs
self._batch_size = batch_size
self._timestep = timestep
def predict(self, stock):
from model_handler import ModelHandler
import matplotlib.dates as mdates
import datetime as dt
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
myModelHandler = ModelHandler()
regressor = myModelHandler.load_json_model(stock)
# Importing the training set
dataset = pd.read_csv(stock.csv_name) #
dates = dataset.iloc[len(dataset) - 31:len(dataset) - 1, 0].values
dates = [dt.datetime.strptime(d, '%Y-%m-%d').date() for d in dates]
# Feature Scaling
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler(feature_range=(0, 1))
dataset_test = dataset[len(dataset) - 30:]
real_stock_price = dataset_test.iloc[:, 1:2].values
# Getting the predicted stock price
dataset = dataset['Open']
inputs = dataset[len(dataset) - len(dataset_test) - 60:].values
inputs = inputs.reshape(-1, 1)
inputs = sc.fit_transform(inputs)
X_test = []
for i in range(60, 90):
X_test.append(inputs[i - 60:i, 0])
X_test = np.array(X_test)
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
predicted_stock_price = regressor.predict(X_test)
print(real_stock_price)
predicted_stock_price = sc.inverse_transform(predicted_stock_price)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
plt.plot(dates,
real_stock_price,
color='red',
label=f'Actual {stock.ticker} Stock Price')
plt.plot(dates,
predicted_stock_price,
color='blue',
label=f'Predicted {stock.ticker} Stock Price')
plt.gcf().autofmt_xdate()
plt.title(f'{stock.ticker} Stock Price Prediction')
plt.xlabel('Time')
plt.ylabel(f'{stock.ticker} Stock Price')
plt.legend()
plt.show()
def forecast(self, stock):
"""Loads the trained model, forecasts the values then displays a graph of the values"""
# Load the trained model
model_handler = ModelHandler()
model = model_handler.load_json_model(stock)
# Importing the training set
dataset = pd.read_csv(stock.csv_name)
dates = dataset.iloc[len(dataset) - 31:len(dataset) - 1, 0].values
dates = [dt.datetime.strptime(d, '%Y-%m-%d').date() for d in dates]
# Create the test dataset
dataset_test = dataset[len(dataset) - 30:]
real_stock_price = dataset_test.iloc[:, 1:2].values
dataset = dataset['Open']
inputs = dataset[len(dataset) - len(dataset_test) - 60:].values
inputs = inputs.reshape(-1, 1)
# Feature Scaling
sc = MinMaxScaler(feature_range=(0, 1))
inputs = sc.fit_transform(inputs)
x_test = []
x_test.append(inputs[0:60, 0])
predicted_values = []
for i in range(1, 31):
x_test_np = np.array(x_test)
x_test_np = np.reshape(x_test_np,
(x_test_np.shape[0], x_test_np.shape[1], 1))
new_data = model.predict(x_test_np)
predicted_values.append(new_data[0])
x_test[0] = np.delete(x_test[0], 0)
x_test[0] = np.concatenate([x_test[0], new_data[0]])
predicted_values = sc.inverse_transform(predicted_values)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
plt.plot(dates,
real_stock_price,
color='red',
label=f'Actual {stock.ticker} Stock Price')
plt.plot(dates,
predicted_values,
color='blue',
label=f'Predicted {stock.ticker} Stock Price')
plt.gcf().autofmt_xdate()
plt.title(f'{stock.ticker} Stock Price Prediction')
plt.xlabel('Time')
plt.ylabel(f'{stock.ticker} Stock Price')
plt.legend()
plt.show()
def init_context(context):
context.logger.info("Init context... 0%")
model = ModelHandler()
setattr(context.user_data, 'model', model)
context.logger.info("Init context...100%")
def init_context(context):
context.logger.info("Init context... 0%")
model = ModelHandler()
context.user_data.model = model
context.logger.info("Init context...100%")
def fileLoad(self):
self.filename = filedialog.askopenfilename()
if self.filename != '':
self.lb.config(text="File selected: " + self.filename)
self.modelHandler1 = ModelHandler(self.filename)
self.modelDrawer()
else:
self.lb.config(text="No file is selected.")
def test_handle(model_hdlr: ModelHandler,
rqst: Request,
context: Context,
num_samples: int,
quantiles: List[str]):
model_hdlr.initialize(context)
results_list = model_hdlr.handle([rqst], context)
# Make sure each JSON line is somewhat correct.
# for each request
for results_list_request in results_list:
for line in io.StringIO(results_list_request):
d = json.loads(line)
for quantile in quantiles:
assert quantile in d["quantiles"]
assert len(d["quantiles"][quantile]) == model_hdlr.mx_model.prediction_length
assert ("mean" in d) and (len(d["mean"]) == model_hdlr.mx_model.prediction_length)
# Print results; need to run pytest -v -rA --tb=short ...
print(results_list)
return results_list
def init_context(context):
context.logger.info("Init context... 0%")
# Read labels
functionconfig = yaml.safe_load(open("/opt/nuclio/function.yaml"))
labels_spec = functionconfig['metadata']['annotations']['spec']
labels = {item['id']: item['name'] for item in json.loads(labels_spec)}
# Read the DL model
model = ModelHandler(labels)
setattr(context.user_data, 'model', model)
context.logger.info("Init context...100%")
class Input(object):
def __init__(self, model_file, syllable_file, syl2chinese):
self.modelhandler = ModelHandler(model_file)
self.syllablehandler = SyllableHandler(syllable_file)
self.lexihandler = LexiHandler(syl2chinese)
self.Init()
def Init(self):
self.syllablehandler.Init()
self.lexihandler.Init()
self.modelhandler.Init()
def SpellSplit(self, pinyin):
return self.syllablehandler.SpellSplit(pinyin)
def SpellToChi(self, spells):
"""
改进字典树 拼音转汉字
"""
results = list()
ch = self.lexihandler.GetChineseFromSpells(spells)
if ch:
results.append(ch)
else:
#没找到 需要在单个字符查找
for i in spells:
res = self.lexihandler.GetChineseFromSpells([i,])
if res:
results.append(res)
return results
def GetChineseFromPinyin(self, pinyin):
spelllist = self.SpellSplit(pinyin)
return self.SpellToChi(spelllist)
def GetProba(self, f, s):
return self.modelhandler.GetChineseProb(f,s)
def init_context(context):
context.logger.info("Init context... 0%")
# Read labels
with open("/opt/nuclio/function.yaml", 'rb') as function_file:
functionconfig = yaml.safe_load(function_file)
labels_spec = functionconfig['metadata']['annotations']['spec']
labels = {item['id']: item['name'] for item in json.loads(labels_spec)}
# Read the DL model
model = ModelHandler(labels)
context.user_data.model = model
context.logger.info("Init context...100%")
def handle(self, **kwargs):
cqfs = kwargs['cqfs']
handler_2 = ModelHandler(successor=None)
handler_1 = ValidateVectorHandler(successor=handler_2)
try:
weight_path = cqfs.dlg.ignicao_line_entrada_formula.text()
if not os.path.isfile(weight_path):
raise RuntimeError('Weight file is mandatory.')
with open(weight_path, 'r') as stream:
weight = yaml.load(stream)
if weight is None or len(weight) == 0:
raise RuntimeError('Fail to load weight file.')
kwargs['weight'] = weight
handler_1.handle(**kwargs)
except Exception as e:
error_msg = u'{} => {}'.format(unicode(e),
unicode(traceback.format_exc()))
self.logger.fatal(error_msg)
cqfs.show_error(error_msg)
def compute_bert_embedding( ):
args = coqa_parser( )
handler = ModelHandler(args)
def main(args):
print_config(args)
set_random_seed(args['random_seed'])
model = ModelHandler(args)
model.train()
model.test()
def __init__(self, model_file, syllable_file, syl2chinese):
self.modelhandler = ModelHandler(model_file)
self.syllablehandler = SyllableHandler(syllable_file)
self.lexihandler = LexiHandler(syl2chinese)
self.Init()
def train(mode, base, set_epochs=30):
print("===== mode: {} | base: {} =====".format(mode, base))
cwd = os.getcwd()
print("current working dir: ", cwd)
cnn_dir = os.path.dirname(cwd)
if mode == 'integrated':
data_dir = os.path.join(cnn_dir, "dogs_vs_cats_integrated")
elif mode == 'native':
data_dir = os.path.join(cnn_dir, "dogs_vs_cats_smaller")
train_dir = os.path.join(data_dir, "train") # global
validation_dir = os.path.join(data_dir, "validation") # global
print("train data is in ... ", train_dir)
print("validation data is in ...", validation_dir)
# make log dir -----
log_dir = os.path.join(cwd, 'comp_log')
os.makedirs(log_dir, exist_ok=True)
child_log_dir = os.path.join(log_dir, "{}_{}".format(mode, base))
os.makedirs(child_log_dir, exist_ok=True)
dh = DaHandler()
train_generator = dh.dataGeneratorFromDir(target_dir=train_dir)
validation_generator = dh.dataGeneratorFromDir(target_dir=validation_dir)
data_checker, label_checker = next(train_generator)
print("data_checker shape : ", data_checker.shape)
print("label_checker shape : ", label_checker.shape)
INPUT_SIZE = data_checker.shape[1]
print("INPUT_SIZE: ", INPUT_SIZE)
CHANNEL = data_checker.shape[3]
print("set channel : ", CHANNEL)
batch_size = data_checker.shape[0]
print("batch_size : ", batch_size)
mh = ModelHandler(INPUT_SIZE, CHANNEL)
if base == 'mymodel':
model = mh.buildMyModel()
elif base == 'mnv1':
model = mh.buildTlearnModel(base='mnv1')
model.summary()
steps_per_epoch = train_generator.n // batch_size
validation_steps = validation_generator.n // batch_size
print(steps_per_epoch, " [steps / epoch]")
print(validation_steps, " (validation steps)")
if mode == 'native':
set_epochs *= 2
history = model.fit_generator(train_generator,
steps_per_epoch=steps_per_epoch,
epochs=set_epochs,
validation_data=validation_generator,
validation_steps=validation_steps,
verbose=1)
# save model & weights
model_file = os.path.join(child_log_dir,
'{}_{}_model.h5'.format(mode, base))
model.save(model_file)
# save history
history_file = os.path.join(child_log_dir,
'{}_{}_history.pkl'.format(mode, base))
with open(history_file, 'wb') as p:
pickle.dump(history.history, p)
print("\nexport logs in ", child_log_dir)
# return 処理 -----
acc_list = history.history['accuracy']
last_acc = acc_list[len(acc_list) - 1]
print("\nlast accuracy: ", last_acc)
val_acc_list = history.history['val_accuracy']
last_val_acc = val_acc_list[len(val_acc_list) - 1]
print("last validation accuracy: ", last_val_acc)
return last_acc, last_val_acc
sys.path.append(os.pardir)
# -----
from da_handler import DaHandler
dah = DaHandler()
aug_list = dah.imgaug_mode_list
aug_list.pop(0)
# -----
from data_handler import DataHandler
dth = DataHandler()
# -----
from model_handler import ModelHandler
mh = ModelHandler(224, 3) # input_size=224, ch=3(共通)
# -----
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
from keras.callbacks import EarlyStopping
# -----
def data_create():
print("Do while below -----\n", aug_list)
for aug in aug_list:
parser.add_argument('--batch-size', type=int, default=2)
parser.add_argument('--shuffle', type=str2bool, default=True)
parser.add_argument('--max_epochs', type=int, default=20)
parser.add_argument('--lr', type=float, default=2e-4)
parser.add_argument('--grad_clip', type=float, default=1.0)
parser.add_argument('--verbose',
type=int,
default=200,
help="print after verbose epochs")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=2,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
args = vars(parser.parse_args())
if args['model_name'] == 'SpanBERT':
download_model()
args['model_path'] = 'tmp_'
# TODO: cuda check
handler = ModelHandler(args)
handler.train()
def test_preprocess(model_hdlr: ModelHandler,
rqst: Request):
results = model_hdlr.preprocess([rqst])
assert len(results) == 1
def test_initialize(model_hdlr: ModelHandler,
context: Context):
model_hdlr.initialize(context)
predictor = model_hdlr.mx_model
assert isinstance(predictor, Predictor)
def test_load_model(model_hdlr: ModelHandler):
predictor = model_hdlr.load_model("../models/DeepAREstimator")
assert isinstance(predictor, Predictor)
def model_hdlr():
model_hdlr = ModelHandler()
return model_hdlr
class LSTMModel:
def __init__(self, epochs=None, batch_size=None, timestep=None, save=True):
self.save = save
self.myModelHandler = ModelHandler()
# Hyperparameters: User chosen parameters
self._epochs = epochs
self._batch_size = batch_size
self._timestep = timestep
# Properties
@property
def epochs(self):
return self._epochs
@epochs.setter
def epochs(self, x):
self._epochs = x
@property
def batch_size(self):
return self._batch_size
@batch_size.setter
def batch_size(self, x):
self._batch_size = x
@property
def timestep(self):
return self._timestep
@timestep.setter
def timestep(self, x):
self._timestep = x
def create_network(self, x_train, y_train, stock):
"""Creates the network, adds layers then trains the network."""
model = Sequential()
model.add(
cLSTM(units=50,
return_sequences=True,
input_shape=(x_train.shape[1], 1)))
model.add(Dropout(0.2))
model.add(cLSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(cLSTM(units=50, return_sequences=True))
model.add(Dropout(0.2))
model.add(cLSTM(units=50))
model.add(Dropout(0.2))
model.add(Dense(units=1))
model.compile(optimizer='adam', loss='mean_squared_error')
model.fit(x_train,
y_train,
epochs=self.epochs,
batch_size=self.batch_size) # Trains the model
self.myModelHandler.save_model_as_json(
model, stock) # Saves the model as .json and weights as .h5
def prepare_dataset(self, stock):
"""Reads the stock CSV then creates the training datasets."""
dataset = pd.DataFrame(yf.download(stock))
dataset_train = dataset[
0:len(dataset) -
30] # Training dataset will be the whole dataset minus the test dataset
training_set = dataset_train.iloc[:, 1:
2].values # Use the open values only
# Feature Scaling
sc = MinMaxScaler(feature_range=(
0, 1)) # Scales all values so that they are in the range (0, 1)
training_set_scaled = sc.fit_transform(training_set)
# Create an input data structure with a timestep
x_train = []
y_train = []
for i in range(self.timestep, len(dataset) - 30):
x_train.append(training_set_scaled[i - self.timestep:i, 0])
y_train.append(training_set_scaled[i, 0])
x_train, y_train = np.array(x_train), np.array(y_train)
# Reshaping
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
return x_train, y_train
def testprepare_dataset(self, stock):
"""Reads the stock CSV then creates the training datasets."""
dataset = pd.DataFrame(yf.download(stock))
dataset = dataset.set_index('Date').diff()
dataset = dataset.drop[dataset.index[0]] # Drops the first null value
dataset_train = dataset[
0:len(dataset) -
30] # Training dataset will be the whole dataset minus the test dataset
training_set = dataset_train.iloc[:, 1:
2].values # Use the open values only
# Feature Scaling
sc = MinMaxScaler(feature_range=(
0, 1)) # Scales all values so that they are in the range (0, 1)
training_set_scaled = sc.fit_transform(training_set)
# Create an input data structure with a timestep
x_train = []
y_train = []
for i in range(self.timestep, len(dataset) - 30):
x_train.append(training_set_scaled[i - self.timestep:i, 0])
y_train.append(training_set_scaled[i, 0])
x_train, y_train = np.array(x_train), np.array(y_train)
# Reshaping
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
return x_train, y_train
def forecast(self, stock):
"""Loads the trained model, forecasts the values then displays a graph of the values"""
# Load the trained model
model_handler = ModelHandler()
model = model_handler.load_json_model(stock)
# Importing the training set
dataset = pd.read_csv(stock.csv_name)
dates = dataset.iloc[len(dataset) - 31:len(dataset) - 1, 0].values
dates = [dt.datetime.strptime(d, '%Y-%m-%d').date() for d in dates]
# Create the test dataset
dataset_test = dataset[len(dataset) - 30:]
real_stock_price = dataset_test.iloc[:, 1:2].values
dataset = dataset['Open']
inputs = dataset[len(dataset) - len(dataset_test) - 60:].values
inputs = inputs.reshape(-1, 1)
# Feature Scaling
sc = MinMaxScaler(feature_range=(0, 1))
inputs = sc.fit_transform(inputs)
x_test = []
x_test.append(inputs[0:60, 0])
predicted_values = []
for i in range(1, 31):
x_test_np = np.array(x_test)
x_test_np = np.reshape(x_test_np,
(x_test_np.shape[0], x_test_np.shape[1], 1))
new_data = model.predict(x_test_np)
predicted_values.append(new_data[0])
x_test[0] = np.delete(x_test[0], 0)
x_test[0] = np.concatenate([x_test[0], new_data[0]])
predicted_values = sc.inverse_transform(predicted_values)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
plt.plot(dates,
real_stock_price,
color='red',
label=f'Actual {stock.ticker} Stock Price')
plt.plot(dates,
predicted_values,
color='blue',
label=f'Predicted {stock.ticker} Stock Price')
plt.gcf().autofmt_xdate()
plt.title(f'{stock.ticker} Stock Price Prediction')
plt.xlabel('Time')
plt.ylabel(f'{stock.ticker} Stock Price')
plt.legend()
plt.show()
def predict(self, stock):
from model_handler import ModelHandler
import matplotlib.dates as mdates
import datetime as dt
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
myModelHandler = ModelHandler()
regressor = myModelHandler.load_json_model(stock)
# Importing the training set
dataset = pd.read_csv(stock.csv_name) #
dates = dataset.iloc[len(dataset) - 31:len(dataset) - 1, 0].values
dates = [dt.datetime.strptime(d, '%Y-%m-%d').date() for d in dates]
# Feature Scaling
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler(feature_range=(0, 1))
dataset_test = dataset[len(dataset) - 30:]
real_stock_price = dataset_test.iloc[:, 1:2].values
# Getting the predicted stock price
dataset = dataset['Open']
inputs = dataset[len(dataset) - len(dataset_test) - 60:].values
inputs = inputs.reshape(-1, 1)
inputs = sc.fit_transform(inputs)
X_test = []
for i in range(60, 90):
X_test.append(inputs[i - 60:i, 0])
X_test = np.array(X_test)
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
predicted_stock_price = regressor.predict(X_test)
print(real_stock_price)
predicted_stock_price = sc.inverse_transform(predicted_stock_price)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
plt.plot(dates,
real_stock_price,
color='red',
label=f'Actual {stock.ticker} Stock Price')
plt.plot(dates,
predicted_stock_price,
color='blue',
label=f'Predicted {stock.ticker} Stock Price')
plt.gcf().autofmt_xdate()
plt.title(f'{stock.ticker} Stock Price Prediction')
plt.xlabel('Time')
plt.ylabel(f'{stock.ticker} Stock Price')
plt.legend()
plt.show()
