def make_multi_model(input_model,
from_config=False,
config_path=None,
weights=None,
prefix=None,
batch_size=8,
lookback=19,
lr=1.52e-5,
ignore_nans=True,
**kwargs):
data_config, nn_config, total_intervals = make_model(
batch_size=batch_size,
lookback=lookback,
lr=lr,
ignore_nans=ignore_nans,
**kwargs)
data_config['inputs'] = [
'tmin', 'tmax', 'slr', 'FLOW_INcms', 'SED_INtons', 'WTEMP(C)',
'CBOD_INppm', 'DISOX_Oppm', 'H20VOLUMEm3'
]
data_config['outputs'] = [
'obs_chla_1',
'obs_chla_3',
'obs_chla_8' # , 'obs_chla_12'
]
fpath = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data')
df_1 = pd.read_csv(os.path.join(fpath, 'data_1.csv'))
df_3 = pd.read_csv(os.path.join(fpath, 'data_3.csv'))
df_8 = pd.read_csv(os.path.join(fpath, 'data_8.csv'))
# df_12 = pd.read_csv(os.path.join(fpath, 'data_12.csv'))
df_1.index = pd.to_datetime(df_1['date'])
df_3.index = pd.to_datetime(df_3['date'])
df_8.index = pd.to_datetime(df_8['date'])
# df_12.index = pd.to_datetime(df_12['date'])
if from_config:
_model = input_model.from_config(
config_path=config_path,
data=[
df_1,
df_3,
df_8 # , df_12
])
_model.build_nn()
_model.load_weights(weights)
else:
_model = input_model(
data_config=data_config,
nn_config=nn_config,
data=[
df_1,
df_3,
df_8 # , df_12
],
prefix=prefix)
return _model
def make_and_run(input_model, _layers=None, lookback=12, epochs=1, **kwargs):
data_config, nn_config, _ = make_model(batch_size=64,
lookback=lookback,
lr=0.001,
epochs=epochs,
**kwargs)
nn_config['layers'] = _layers
file_path = abspath(getsourcefile(lambda: 0))
dpath = os.path.join(os.path.dirname(os.path.dirname(file_path)), "data")
fname = os.path.join(dpath, "nasdaq100_padding.csv")
df = pd.read_csv(fname)
model = input_model(
data_config=data_config,
nn_config=nn_config,
data=df,
verbosity=0
)
model.build_nn()
_ = model.train_nn(indices='random')
_, pred_y = model.predict(use_datetime_index=False)
return pred_y
def build_model(**kwargs):
data_config, nn_config, _ = make_model(batch_size=batch_size,
lookback=lookback,
normalize=False,
epochs=1,
**kwargs)
model = Model(data_config, nn_config, data=data, verbosity=0)
model.build_nn()
return model
def make_and_run(model,
lookback=12,
epochs=4,
batch_size=16,
data_type='other',
**kwargs):
if data_type == "nasdaq":
fname = os.path.join(dpath, "nasdaq100_padding.csv")
df = pd.read_csv(fname)
in_cols = list(df.columns)
in_cols.remove("NDX")
inputs = in_cols
outputs = ["NDX"]
else:
fname = os.path.join(dpath, "data_30min.csv")
df = pd.read_csv(fname)
inputs = input_features
# column in dataframe to bse used as output/target
outputs = ['target7']
data_config, nn_config, total_intervals = make_model(batch_size=batch_size,
lookback=lookback,
lr=0.001,
inputs=inputs,
outputs=outputs,
epochs=epochs,
**kwargs)
model = model(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=None if data_type == "nasdaq" else total_intervals,
verbosity=0)
model.build_nn()
_ = model.train_nn(indices='random')
_, pred_y = model.predict(use_datetime_index=False)
return pred_y
import os
from dl4seq.utils import make_model
from dl4seq import Model
if __name__ == "__main__":
input_features = [
'input1', 'input2', 'input3', 'input4', 'input5', 'input6', 'input8',
'input11'
]
# column in dataframe to bse used as output/target
outputs = ['target7']
data_config, nn_config, total_intervals = make_model(batch_size=16,
lookback=1,
inputs=input_features,
outputs=outputs,
lr=0.0001)
fname = os.path.join(os.path.dirname(os.path.dirname(__file__)),
"data\\nasdaq100_padding.csv")
df = pd.read_csv(fname)
df.index = pd.to_datetime(df['Date_Time2'])
model = Model(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=total_intervals)
model.build_nn()
from dl4seq.pytorch_models import IMVLSTMModel
from dl4seq.utils import make_model
import pandas as pd
lookback = 10
epochs = 50
df = pd.read_csv("../data/nasdaq100_padding.csv")
data_config, nn_config, total_intervals = make_model(batch_size=16,
lookback=lookback,
lr=0.001,
use_predicted_output=True,
epochs=epochs)
model = IMVLSTMModel(data_config=data_config, nn_config=nn_config, data=df)
model.build_nn()
h = model.train_nn(st=0, en=1000)
x, y = model.predict(st=0, en=1000)
model.plot_activations()
return
if __name__ == "__main__":
input_features = [
'input1', 'input2', 'input3', 'input4', 'input5', 'input6', 'input8',
'input11'
]
# column in dataframe to bse used as output/target
outputs = ['target7', 'target8']
data_config, nn_config, total_intervals = make_model(
batch_size=4,
lookback=15,
inputs=input_features,
outputs=outputs,
lr=0.0001,
epochs=20,
val_fraction=0.3, # TODO why less than 0.3 give error here?
test_fraction=0.3)
fname = os.path.join(os.path.dirname(os.path.dirname(__file__)),
"data\\nasdaq100_padding.csv")
df = pd.read_csv(fname)
df.index = pd.to_datetime(df['Date_Time2'])
model = MultiSite(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=total_intervals)
print(out.isna().sum())
# put four chunks of missing intervals
intervals = [(100, 200), (1000, 8000), (10000, 31000)]
for interval in intervals:
st, en = interval[0], interval[1]
out[st:en] = np.nan
df["NDX"] = out
print("{} nan values created in NDX column".format(out.isna().sum()))
# verify the missing values
print(df[98:108])
data_config, nn_config, _ = make_model(batch_size=32, lookback=5, lr=0.0001)
model = DualAttentionModel(data_config=data_config,
nn_config=nn_config,
data=df,
intervals=[(0, 99), (200, 999), (8000, 9999),
(31000, 40560)])
model.build_nn()
history = model.train_nn(indices='random')
y, obs = model.predict(indices=model.test_indices, use_datetime_index=False)
# tr_y, tr_obs = model.predict(indices=model.train_indices, pref='train', use_datetime_index=False)
model.view_model(st=0, save=True) # takes a lot of time to save all plots
# this file tests that given activations are working both as layers as well as activation functions withing a layer
import unittest
import os
import site # so that dl4seq directory is in path
site.addsitedir(os.path.dirname(os.path.dirname(__file__)) )
from dl4seq import Model
from dl4seq.utils import make_model
import pandas as pd
data_config, nn_config, _ = make_model()
nn_config['epochs'] = 2
data_config['lookback'] = 1
from inspect import getsourcefile
from os.path import abspath
file_path = abspath(getsourcefile(lambda:0))
dpath = os.path.join(os.path.dirname(os.path.dirname(file_path)), "data")
fname = os.path.join(dpath, "nasdaq100_padding.csv")
df = pd.read_csv(fname)
class TestActivations(unittest.TestCase):
def test_as_layers(self):
layers = {}
