def load_model(): model_save_path = constant.load_model_path state = torch.load(model_save_path, map_location=lambda storage, location: storage) constant.arg = state['config'] load_settings() if constant.model == "LSTM": model = LstmModel(vocab=vocab, embedding_size=constant.emb_dim, hidden_size=constant.hidden_dim, num_layers=constant.n_layers, is_bidirectional=constant.bidirec, input_dropout=constant.drop, layer_dropout=constant.drop, attentive=constant.attn) elif constant.model == "UTRS": model = UTransformer(vocab=vocab, embedding_size=constant.emb_dim, hidden_size=constant.hidden_dim, num_layers=constant.hop, num_heads=constant.heads, total_key_depth=constant.depth, total_value_depth=constant.depth, filter_size=constant.filter, act=constant.act) elif constant.model == "ELMO": model = ELMoEncoder(C=4) else: print("Model is not defined") exit(0) model = model.load_state_dict(state['model']) return model
def rebalance_porfolio(date, asset, companies_data): # date_string = datetime.datetime.strftime(date, "%Y-%m-%d") print("=" * 50) print("Rebalancing Portfolio For Current Date: ", date) print("=" * 50) # Load Trained Model lstm = LstmModel(name="Test2", load=True) # Save Values In Dict companies_rtv = dict() companies_prices = dict() # Predict One-Week Price # Calculate Return-To-Volatility # Save The Result For Different Companies day_in, day_want, step = 50, 10, 2 for company, data in companies_data.items(): # Predictions From LSTM idx = data[data['Date'] == date].index.item() _input = data.iloc[idx - day_in:idx + day_want - step + 1, 1:data.shape[1]] predictions = pd.DataFrame(lstm.predict(_input), index=pd.date_range(start=date, periods=10)) # Calculate Return-To-Volatility rtv = calculate_rtv(predictions) companies_rtv[company] = rtv # Get Price List companies_prices[company] = data.iloc[idx, 5] # Sort The Results Descending & Choose Top 10 Companies filtered_list = list(filter(lambda x: x[1] >= 0, companies_rtv.items())) sorted_list = sorted(filtered_list, key=lambda x: x[1], reverse=True) companies_rtv_filter = {t[0]: t[1] for t in sorted_list[:10]} # Build Portfolio rtv_sum = sum(companies_rtv_filter.values()) portfolio_pct = { key: (value / rtv_sum) for key, value in companies_rtv_filter.items() } portfolio = { company: ((asset * pct) / companies_prices[company]) for company, pct in portfolio_pct.items() } print("Company Prices: ", companies_prices) print("Companies Return-To-Volatility: ", companies_rtv) print("Filtered 10 Companies: ", companies_rtv_filter) print("Portfolio Percentage: ", portfolio_pct) print("Portfolio: ", portfolio) return portfolio, portfolio_pct
def get_baseline_model(args): vocab = utils.load_vocab(args.vocab_json) if args.baseline_start_from is not None: model, kwargs = utils.load_model(args.baseline_start_from) elif args.model_type == 'LSTM': kwargs = { 'vocab': vocab, 'rnn_wordvec_dim': args.rnn_wordvec_dim, 'rnn_dim': args.rnn_hidden_dim, 'rnn_num_layers': args.rnn_num_layers, 'rnn_dropout': args.rnn_dropout, 'fc_dims': parse_int_list(args.classifier_fc_dims), 'fc_use_batchnorm': args.classifier_batchnorm == 1, 'fc_dropout': args.classifier_dropout, } model = LstmModel(**kwargs) elif args.model_type == 'CNN+LSTM': kwargs = { 'vocab': vocab, 'rnn_wordvec_dim': args.rnn_wordvec_dim, 'rnn_dim': args.rnn_hidden_dim, 'rnn_num_layers': args.rnn_num_layers, 'rnn_dropout': args.rnn_dropout, 'cnn_feat_dim': parse_int_list(args.feature_dim), 'cnn_num_res_blocks': args.cnn_num_res_blocks, 'cnn_res_block_dim': args.cnn_res_block_dim, 'cnn_proj_dim': args.cnn_proj_dim, 'cnn_pooling': args.cnn_pooling, 'fc_dims': parse_int_list(args.classifier_fc_dims), 'fc_use_batchnorm': args.classifier_batchnorm == 1, 'fc_dropout': args.classifier_dropout, } model = CnnLstmModel(**kwargs) elif args.model_type == 'CNN+LSTM+SA': kwargs = { 'vocab': vocab, 'rnn_wordvec_dim': args.rnn_wordvec_dim, 'rnn_dim': args.rnn_hidden_dim, 'rnn_num_layers': args.rnn_num_layers, 'rnn_dropout': args.rnn_dropout, 'cnn_feat_dim': parse_int_list(args.feature_dim), 'stacked_attn_dim': args.stacked_attn_dim, 'num_stacked_attn': args.num_stacked_attn, 'fc_dims': parse_int_list(args.classifier_fc_dims), 'fc_use_batchnorm': args.classifier_batchnorm == 1, 'fc_dropout': args.classifier_dropout, } model = CnnLstmSaModel(**kwargs) if model.rnn.token_to_idx != vocab['question_token_to_idx']: # Make sure new vocab is superset of old for k, v in model.rnn.token_to_idx.items(): assert k in vocab['question_token_to_idx'] assert vocab['question_token_to_idx'][k] == v for token, idx in vocab['question_token_to_idx'].items(): model.rnn.token_to_idx[token] = idx kwargs['vocab'] = vocab model.rnn.expand_vocab(vocab['question_token_to_idx']) model.cuda() model.train() return model, kwargs
def dummy_porfolio(date, asset, companies_data): lstm = LstmModel(name="Test2", load=True) day_in, day_want, step = 50, 10, 2 for company, data in companies_data.items(): # Predictions From LSTM idx = data[data['Date'] == date].index.item() print(data.iloc[idx - day_in:idx + day_want - step + 1, 0:data.shape[1]]) _input = data.iloc[idx - day_in:idx + day_want - step + 1, 1:data.shape[1]] predictions = pd.DataFrame(lstm.predict(_input), index=pd.date_range(start=date, periods=10)) break return {}, {}
def load_model(): model_load_path = constant.load_model_path model_save_path = constant.save_path state = torch.load(model_load_path, map_location=lambda storage, location: storage) arg = state['config'] load_settings(arg) data_loaders_train, data_loaders_val, data_loaders_test, vocab = prepare_data_loaders( num_split=1, batch_size=constant.batch_size, hier=False, elmo=constant.elmo, dev_with_label=False, include_test=True) if constant.model == "LSTM": model = LstmModel(vocab=vocab, embedding_size=constant.emb_dim, hidden_size=constant.hidden_dim, num_layers=constant.n_layers, is_bidirectional=constant.bidirec, input_dropout=constant.drop, layer_dropout=constant.drop, attentive=constant.attn) elif constant.model == "UTRS": model = UTransformer(vocab=vocab, embedding_size=constant.emb_dim, hidden_size=constant.hidden_dim, num_layers=constant.hop, num_heads=constant.heads, total_key_depth=constant.depth, total_value_depth=constant.depth, filter_size=constant.filter, act=constant.act) elif constant.model == "ELMO": model = ELMoEncoder(C=4) else: print("Model is not defined") exit(0) model.load_state_dict(state['model']) return model, data_loaders_test, vocab, model_save_path
import keras import pandas as pd import numpy as np from models import LstmModel from config import DIR_CONFIG, FILE_CONFIG if __name__ == "__main__": # Initialize Dates # START_DATE = '2017-01-02' # END_DATE = '2019-06-01' START_DATE = '2015-07-06' END_DATE = '2018-05-28' # Initialize Model lstm = LstmModel(name='Old2') # Load List Of Companies companies = [] with open(FILE_CONFIG["COMPANY_LIST"], 'r') as f: for line in f: companies.append(line.rstrip("\n")) print(companies) # Train Model With Different Companies Adj Close Values for company in companies: path = os.path.join(DIR_CONFIG["DATA_DIR"], '{}_data.csv'.format(company)) raw_data = pd.read_csv(path, index_col=0) raw_data = raw_data.loc[:END_DATE]["Adj Close"].values raw_data = np.reshape(raw_data, (raw_data.shape[0], 1))
import json # Add the models directory to the path dir_path = os.path.dirname(os.path.realpath(__file__)) parent_dir = os.path.dirname(dir_path) sys.path.append(os.path.join(parent_dir, 'birdsong')) #from models import Zilpzalp #from models import Hawk from models import LstmModel from utils import avg_score, maxwindow_score, get_top5_prediction app = Flask(__name__) # Initiate the model model = LstmModel(time_axis=216, freq_axis=256, no_classes=100) # Load the state of model from checkpoint checkpoint_path = 'model/checkpoint_Lstm_29-03' checkpoint = torch.load(checkpoint_path, map_location='cpu') state = checkpoint['state_dict'] model.load_state_dict(state) model.eval() # Add the dictionary with the species info label_dict = {} reader = csv.DictReader(open('model/top100_codes_translated.csv')) for row in reader: label_dict[int(row['id1'])] = { 'name': row['english'], 'img_source': row['img_source'],
import keras import pandas as pd import numpy from models import LstmModel CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(CURRENT_DIR, 'old_data') CHECKPOINT_DIR = os.path.join(CURRENT_DIR, 'checkpoint') print("Data Stored In", DATA_DIR) print("Checkpoint Stored In", CHECKPOINT_DIR) if __name__ == "__main__": # Initialize Model lstm = LstmModel(name='Test2') # Load List Of Companies companies = [] with open(os.path.join(CURRENT_DIR, "company_list.txt"), 'r') as f: for line in f: companies.append(line.rstrip("\n")) print(companies) # Train Model With Different Companies Data for company in companies: path = os.path.join(DATA_DIR, '{}_data.csv'.format(company)) raw_data = pd.read_csv(path, index_col=0) raw_data = raw_data.loc[:'2017-12-31'].values #print(raw_data)
def rebalance_porfolio(date, asset, companies_data): # date_string = datetime.datetime.strftime(date, "%Y-%m-%d") print("=" * 50) print("Rebalancing Portfolio For Current Date: ", date) print("=" * 50) # Load Trained Model lstm = LstmModel(name="Old1", load=True) # Save Values In Dict companies_rtv = dict() companies_prices = dict() # Predict One-Week Price # Calculate Return-To-Volatility # Save The Result For Different Companies day_in, day_out, num_features = lstm.load_settings() for company, data in companies_data.items(): # Current Date's Index try: idx = data[data['Date'] == date].index.values[0] except IndexError: # When Company Data Is Empty On That Date continue # Predictions From LSTM _input = data.iloc[idx - day_in:idx]['Adj Close'].values _input = np.reshape(_input, (_input.shape[0], 1)) predictions = pd.DataFrame(lstm.predict(company=company, _input=_input), index=pd.date_range(start=date, periods=day_out)) # Calculate Return-To-Volatility rtv = calculate_rtv(predictions) companies_rtv[company] = rtv # Get Adj Closing Price List companies_prices[company] = data.iloc[idx, 4] # Sort The Results Descending & Choose Top 10 Companies filtered_list = list(filter(lambda x: x[1] > 0, companies_rtv.items())) sorted_list = sorted(filtered_list, key=lambda x: x[1], reverse=True) companies_rtv_filter = {t[0]: t[1] for t in sorted_list[:10]} # Build Portfolio rtv_sum = sum(companies_rtv_filter.values()) portfolio_pct = { key: (value / rtv_sum) for key, value in companies_rtv_filter.items() } portfolio = { company: ((asset * pct) / companies_prices[company]) for company, pct in portfolio_pct.items() } print("Company Prices: ", companies_prices) print("Companies Return-To-Volatility: ", companies_rtv) print("Filtered 10 Companies: ", companies_rtv_filter) print("Portfolio Percentage: ", portfolio_pct) print("Portfolio: ", portfolio) return portfolio, portfolio_pct
def main(): logging.basicConfig(level=logging.INFO) task_id = str(int(time.time())) tmp_model_path = os.path.join('/tmp', '%s.h5' % task_id) if True: task = { 'task_id': task_id, 'score_metric': 'val_rmse', 'dataset_path': 'showdown_full', 'final': True, 'model_config': TransferLstmModel.create_cnn( tmp_model_path, transform_model_config={ 'model_uri': '/models/snapshots/regression/1480182349/31.h5', 'scale': 16, 'type': 'regression' }, timesteps=50, W_l2=0.001, scale=16., input_shape=(120, 320, 3)), 'training_args': { 'batch_size': 32, 'epochs': 100, }, } if False: task = { 'task_id': task_id, 'score_metric': 'loss', 'dataset_path': 'shinale_full', 'final': False, 'model_config': RegressionModel.create_resnet_inception_v2(tmp_model_path, learning_rate=0.001, input_shape=(120, 320, 3)), 'training_args': { 'batch_size': 16, 'epochs': 100, 'pctl_sampling': 'uniform', 'pctl_thresholds': showdown_percentiles(), }, } if False: task = { 'task_id': task_id, 'score_metric': 'loss', 'dataset_path': 'showdown_full', 'final': True, 'model_config': { 'model_uri': '/models/output/1480004259.h5', 'scale': 16, 'type': 'regression' }, 'training_args': { 'batch_size': 32, 'epochs': 40, }, } if False: # sharp left vs center vs sharp right task = { 'task_id': task_id, 'dataset_path': 'finale_full', 'score_metric': 'val_categorical_accuracy', 'model_config': CategoricalModel.create(tmp_model_path, use_adadelta=True, W_l2=0.001, thresholds=[-0.061, 0.061]), 'training_args': { 'batch_size': 32, 'epochs': 30, 'pctl_sampling': 'uniform', }, } if False: # half degree model task = { 'task_id': task_id, 'dataset_path': 'finale_center', 'model_config': CategoricalModel.create(tmp_model_path, use_adadelta=True, learning_rate=0.001, thresholds=np.linspace(-0.061, 0.061, 14)[1:-1], input_shape=(120, 320, 3)), 'training_args': { 'pctl_sampling': 'uniform', 'batch_size': 32, 'epochs': 20, }, } if False: input_model_config = { 'model_uri': 's3://sdc-matt/simple/1477715388/model.h5', 'type': 'simple', 'cat_classes': 5 } ensemble_model_config = EnsembleModel.create(tmp_model_path, input_model_config, timesteps=3, timestep_noise=0.1, timestep_dropout=0.5) task = { 'task_id': task_id, 'dataset_path': 'final_training', 'model_config': ensemble_model_config, 'training_args': { 'batch_size': 64, 'epochs': 3 }, } if False: lstm_model_config = LstmModel.create(tmp_model_path, (10, 120, 320, 3), timesteps=10, W_l2=0.0001, scale=60.0) task = { 'task_id': task_id, 'dataset_path': 'showdown_full', 'final': True, 'model_config': lstm_model_config, 'training_args': { 'pctl_sampling': 'uniform', 'batch_size': 32, 'epochs': 10, }, } handle_task(task)
dev_with_label=constant.dev_with_label, include_test=constant.include_test) results = [] for i in range(constant.num_split): data_loader_train = data_loaders_train[i] data_loader_val = data_loaders_val[i] data_loader_test = data_loaders_test[i] print("###### EXPERIMENT {} ######".format(i + 1)) print("(EXPERIMENT %d) Create the model" % (i + 1)) if constant.model == "LSTM": model = LstmModel(vocab=vocab, embedding_size=constant.emb_dim, hidden_size=constant.hidden_dim, num_layers=constant.n_layers, is_bidirectional=constant.bidirec, input_dropout=constant.drop, layer_dropout=constant.drop, attentive=constant.attn) elif constant.model == "UTRS": model = UTransformer(vocab=vocab, embedding_size=constant.emb_dim, hidden_size=constant.hidden_dim, num_layers=constant.hop, num_heads=constant.heads, total_key_depth=constant.depth, total_value_depth=constant.depth, filter_size=constant.filter, act=constant.act) elif constant.model == "ELMO": model = ELMoEncoder(C=4)