def make_multi_model(input_model, from_config=False, config_path=None, weights=None, prefix=None, batch_size=8, lookback=19, lr=1.52e-5, ignore_nans=True, **kwargs): data_config, nn_config, total_intervals = make_model( batch_size=batch_size, lookback=lookback, lr=lr, ignore_nans=ignore_nans, **kwargs) data_config['inputs'] = [ 'tmin', 'tmax', 'slr', 'FLOW_INcms', 'SED_INtons', 'WTEMP(C)', 'CBOD_INppm', 'DISOX_Oppm', 'H20VOLUMEm3' ] data_config['outputs'] = [ 'obs_chla_1', 'obs_chla_3', 'obs_chla_8' # , 'obs_chla_12' ] fpath = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data') df_1 = pd.read_csv(os.path.join(fpath, 'data_1.csv')) df_3 = pd.read_csv(os.path.join(fpath, 'data_3.csv')) df_8 = pd.read_csv(os.path.join(fpath, 'data_8.csv')) # df_12 = pd.read_csv(os.path.join(fpath, 'data_12.csv')) df_1.index = pd.to_datetime(df_1['date']) df_3.index = pd.to_datetime(df_3['date']) df_8.index = pd.to_datetime(df_8['date']) # df_12.index = pd.to_datetime(df_12['date']) if from_config: _model = input_model.from_config( config_path=config_path, data=[ df_1, df_3, df_8 # , df_12 ]) _model.build_nn() _model.load_weights(weights) else: _model = input_model( data_config=data_config, nn_config=nn_config, data=[ df_1, df_3, df_8 # , df_12 ], prefix=prefix) return _model
def make_and_run(input_model, _layers=None, lookback=12, epochs=1, **kwargs): data_config, nn_config, _ = make_model(batch_size=64, lookback=lookback, lr=0.001, epochs=epochs, **kwargs) nn_config['layers'] = _layers file_path = abspath(getsourcefile(lambda: 0)) dpath = os.path.join(os.path.dirname(os.path.dirname(file_path)), "data") fname = os.path.join(dpath, "nasdaq100_padding.csv") df = pd.read_csv(fname) model = input_model( data_config=data_config, nn_config=nn_config, data=df, verbosity=0 ) model.build_nn() _ = model.train_nn(indices='random') _, pred_y = model.predict(use_datetime_index=False) return pred_y
def build_model(**kwargs): data_config, nn_config, _ = make_model(batch_size=batch_size, lookback=lookback, normalize=False, epochs=1, **kwargs) model = Model(data_config, nn_config, data=data, verbosity=0) model.build_nn() return model
def make_and_run(model, lookback=12, epochs=4, batch_size=16, data_type='other', **kwargs): if data_type == "nasdaq": fname = os.path.join(dpath, "nasdaq100_padding.csv") df = pd.read_csv(fname) in_cols = list(df.columns) in_cols.remove("NDX") inputs = in_cols outputs = ["NDX"] else: fname = os.path.join(dpath, "data_30min.csv") df = pd.read_csv(fname) inputs = input_features # column in dataframe to bse used as output/target outputs = ['target7'] data_config, nn_config, total_intervals = make_model(batch_size=batch_size, lookback=lookback, lr=0.001, inputs=inputs, outputs=outputs, epochs=epochs, **kwargs) model = model(data_config=data_config, nn_config=nn_config, data=df, intervals=None if data_type == "nasdaq" else total_intervals, verbosity=0) model.build_nn() _ = model.train_nn(indices='random') _, pred_y = model.predict(use_datetime_index=False) return pred_y
import os from dl4seq.utils import make_model from dl4seq import Model if __name__ == "__main__": input_features = [ 'input1', 'input2', 'input3', 'input4', 'input5', 'input6', 'input8', 'input11' ] # column in dataframe to bse used as output/target outputs = ['target7'] data_config, nn_config, total_intervals = make_model(batch_size=16, lookback=1, inputs=input_features, outputs=outputs, lr=0.0001) fname = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data\\nasdaq100_padding.csv") df = pd.read_csv(fname) df.index = pd.to_datetime(df['Date_Time2']) model = Model(data_config=data_config, nn_config=nn_config, data=df, intervals=total_intervals) model.build_nn()
from dl4seq.pytorch_models import IMVLSTMModel from dl4seq.utils import make_model import pandas as pd lookback = 10 epochs = 50 df = pd.read_csv("../data/nasdaq100_padding.csv") data_config, nn_config, total_intervals = make_model(batch_size=16, lookback=lookback, lr=0.001, use_predicted_output=True, epochs=epochs) model = IMVLSTMModel(data_config=data_config, nn_config=nn_config, data=df) model.build_nn() h = model.train_nn(st=0, en=1000) x, y = model.predict(st=0, en=1000) model.plot_activations()
return if __name__ == "__main__": input_features = [ 'input1', 'input2', 'input3', 'input4', 'input5', 'input6', 'input8', 'input11' ] # column in dataframe to bse used as output/target outputs = ['target7', 'target8'] data_config, nn_config, total_intervals = make_model( batch_size=4, lookback=15, inputs=input_features, outputs=outputs, lr=0.0001, epochs=20, val_fraction=0.3, # TODO why less than 0.3 give error here? test_fraction=0.3) fname = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data\\nasdaq100_padding.csv") df = pd.read_csv(fname) df.index = pd.to_datetime(df['Date_Time2']) model = MultiSite(data_config=data_config, nn_config=nn_config, data=df, intervals=total_intervals)
print(out.isna().sum()) # put four chunks of missing intervals intervals = [(100, 200), (1000, 8000), (10000, 31000)] for interval in intervals: st, en = interval[0], interval[1] out[st:en] = np.nan df["NDX"] = out print("{} nan values created in NDX column".format(out.isna().sum())) # verify the missing values print(df[98:108]) data_config, nn_config, _ = make_model(batch_size=32, lookback=5, lr=0.0001) model = DualAttentionModel(data_config=data_config, nn_config=nn_config, data=df, intervals=[(0, 99), (200, 999), (8000, 9999), (31000, 40560)]) model.build_nn() history = model.train_nn(indices='random') y, obs = model.predict(indices=model.test_indices, use_datetime_index=False) # tr_y, tr_obs = model.predict(indices=model.train_indices, pref='train', use_datetime_index=False) model.view_model(st=0, save=True) # takes a lot of time to save all plots
# this file tests that given activations are working both as layers as well as activation functions withing a layer import unittest import os import site # so that dl4seq directory is in path site.addsitedir(os.path.dirname(os.path.dirname(__file__)) ) from dl4seq import Model from dl4seq.utils import make_model import pandas as pd data_config, nn_config, _ = make_model() nn_config['epochs'] = 2 data_config['lookback'] = 1 from inspect import getsourcefile from os.path import abspath file_path = abspath(getsourcefile(lambda:0)) dpath = os.path.join(os.path.dirname(os.path.dirname(file_path)), "data") fname = os.path.join(dpath, "nasdaq100_padding.csv") df = pd.read_csv(fname) class TestActivations(unittest.TestCase): def test_as_layers(self): layers = {}