def main():
start_time = time.time()
in_arg = args_input()
# get the data
dataset, dataloaders = functions.load_data()
# get the model
model = functions.get_model(in_arg.arch)
# get the classifier, criterion, optimizer, device
model, classifier, criterion, optimizer, device = network_param(
model, in_arg.arch, in_arg.hidden_units, in_arg.learning_rate,
in_arg.gpu)
model.to(device)
print('Training model...')
functions.train_model(model, criterion, optimizer, device,
dataloaders['train'], dataloaders['val'],
in_arg.epochs)
# validation on test data
print('\nGetting results on test data accuracy...')
functions.test_model(model, criterion, device, dataloaders['test'])
# saving checkpoint on trained model
print('\nSaving checkpoint for current trained model...')
functions.save_checkpoint(model, optimizer, dataset, in_arg.arch,
in_arg.epochs, in_arg.save_dir)
print('Checkpoint saved!')
def main():
parser = argparse.ArgumentParser(
description='This program predicts a flower name from an image')
parser.add_argument('data_dir', type=str, help='Dataset directory')
parser.add_argument('--save_dir',
type=str,
default='./',
help='Saved checkpoint directory')
parser.add_argument('--arch',
type=str,
default='vgg16',
help='Network architecture')
parser.add_argument('--hidden_units',
type=int,
default='256',
help='Hidden units')
parser.add_argument('--dropout',
type=float,
default='0.2',
help='Dropout for the hidden layers')
parser.add_argument('--num_classes',
type=int,
default='256',
help='Number of classes for classification')
parser.add_argument('--learning_rate',
type=float,
default='0.005',
help='Learning rate')
parser.add_argument('--epochs',
type=int,
default='20',
help='Number of epochs')
parser.add_argument('--gpu',
action='store_true',
help='Use GPU',
default=False)
args = parser.parse_args()
device = torch.device("cuda:0" if torch.cuda.is_available() else 'cpu'
) if args.gpu else 'cpu'
model, criterion, optimizer, scheduler = initialize_model(
args.arch, args.hidden_units, args.dropout, args.num_classes, device,
args.learning_rate)
dataloaders, image_datasets = create_loaders(args.data_dir)
train_model(model, dataloaders, criterion, optimizer, scheduler,
image_datasets, args.epochs, device)
test_model(model, dataloaders, image_datasets, device)
save_checkpoint(model, optimizer, scheduler, args.epochs,
args.learning_rate, f'{args.arch}_checkpoint.pth')
def full_execution(midi_files, output_tag, total_epochs, batch_size,
sequence_length, temperature, offset_adj):
# epoch_stops = 1
# epoch_count = 0
weight_file = None
note_file = fxn.convert_midis_to_notes(midi_files, output_tag)
epochs = total_epochs
with open(note_file, 'rb') as filepath:
notes = pickle.load(filepath)
network_input, network_output, n_patterns, n_vocab, pitchnames = fxn.prepare_sequences(
notes, sequence_length)
network_input_r, network_output_r = fxn.reshape_for_training(
network_input, network_output, sequence_length)
# while epoch_count <= total_epochs:
# epochs = epoch_stops
#
model = fxn.create_network(network_input_r, n_vocab, weight_file)
model, weight_file, history = fxn.train_model(model, network_input_r,
network_output_r, epochs,
batch_size, output_tag,
sequence_length)
normalized_input = fxn.reshape_for_creation(network_input, n_patterns,
sequence_length, n_vocab)
model = fxn.create_network(normalized_input, n_vocab, weight_file)
prediction_output = fxn.generate_notes(model, network_input, pitchnames,
sequence_length, notes_generated,
n_vocab, temperature)
output_notes = fxn.create_midi(prediction_output, output_tag,
sequence_length, offset_adj)
# epoch_count += epoch_stops
return output_notes, history, weight_file
def main():
start_time = time()
in_arg = get_input_args()
#check_command_line_arguments(in_arg)
images_datasets = data_loaders(in_arg.data_dir)
if in_arg.gpu is True:
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
print("Using CPU since GPU is not available")
else:
device = torch.device('cpu')
model = getattr(models, in_arg.arch)(pretrained=True)
for param in model.features.parameters():
param.require_grad = False
output_classes = 102
hidden_units = 2000
if in_arg.arch == 'vgg16':
classifier = nn.Sequential(
OrderedDict([('fc1', nn.Linear(25088, 4096)), ('relu', nn.ReLU()),
('dropout', nn.Dropout(p=0.5)),
('fc2', nn.Linear(4096, in_arg.hidden_units)),
('relu', nn.ReLU()), ('dropout', nn.Dropout(p=0.5)),
('fc3', nn.Linear(in_arg.hidden_units,
output_classes))]))
model.classifier = classifier
if in_arg.arch == 'alexnet':
classifier = nn.Sequential(
OrderedDict([('dropout', nn.Dropout(p=0.5)),
('fc1', nn.Linear(9216, 4096)), ('relu', nn.ReLU()),
('dropout', nn.Dropout(p=0.5)),
('fc2', nn.Linear(4096, in_arg.hidden_units)),
('relu', nn.ReLU()),
('fc3', nn.Linear(in_arg.hidden_units,
output_classes))]))
model.classifier = classifier
model = model.to(device)
criterion = nn.CrossEntropyLoss()
l_r = in_arg.learning_rate
epochs = in_arg.epochs
optimizer = optim.SGD(model.classifier.parameters(), lr=l_r, momentum=0.9)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)
model = train_model(model, criterion, optimizer, images_datasets,
exp_lr_scheduler, device, epochs)
model.class_to_idx = images_datasets['train'].class_to_idx
checkpoint = {
'class_to_idx': model.class_to_idx,
'arch': in_arg.arch,
'classifier': model.classifier,
'state_dict': model.state_dict(),
'epochs': in_arg.epochs,
'learning_rate': in_arg.learning_rate,
'hidden_units': in_arg.hidden_units
}
torch.save(checkpoint, in_arg.save_dir)
end_time = time()
def main():
args = parser_fun_train()
with open('cat_to_name.json', 'r') as f:
cat_to_name = json.load(f)
#training the model
optimizer, model = train_model(args.arch, imd.train_dl, imd.valid_dl,
args.epochs, 40, args.device,
args.hidden_units, args.lr)
create_checkpoint(model, '', args.arch, imd.training_dataset.class_to_idx,
optimizer, args.epochs)
def train_model_launcher(r, w, tP, u_ed, xi, save_results=False, i=''):
bayes_opt_score = functions.train_model(
r,
w,
tP,
u_ed,
xi,
save_results=save_results,
i=i,
config_parameters=config_parameters,
fileName=fileName)
return bayes_opt_score
def main():
#print("hello world") for luck
in_arg = train_input_args()
print(" data directory =",in_arg.data_dir, "\n save directory =", in_arg.save_dir, "\n model architecture =", in_arg.arch,
"\n hidden units =", in_arg.hidden_units, "\n learning rate =", in_arg.learning_rate,
"\n epochs =", in_arg.epochs, "\n device =", in_arg.device)
image_data, data_loader = load_images()
model, optimizer, criterion = build_model(in_arg.arch , in_arg.hidden_units, in_arg.learning_rate)
model = train_model(model,data_loader, in_arg.epochs, criterion, optimizer, in_arg.device)
save_model(model, optimizer, in_arg.arch, in_arg.data_dir, in_arg.save_dir,image_data)
print("-"*40)
def main():
# Get CLI arguments
args = get_input_args()
# Prep data
train_transform = utilities.transform_data('train')
test_transform = utilities.transform_data('test')
# Dataloaders
trainloader = utilities.load_data(args.data_directory + '/' + 'train',
train_transform)
validationloader = utilities.load_data(args.data_directory + '/' + 'valid',
test_transform)
# Setup and train model
model, optimizer, criterion = functions.model_setup(
args.arch, args.hidden_units, args.learning_rate)
trained_model = functions.train_model(optimizer, criterion, model,
trainloader, validationloader,
args.gpu, args.epochs)
# Save the model
functions.save_checkpoint(trained_model, args.save_dir)
import torch
from model import Net
from functions import load_data, train_model, test_model, print_prediction
lr = 0.001
if __name__ == '__main__':
train, validation = load_data()
cuda = torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")
model = Net().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# train the model
train_model(train, optimizer, model, device)
# test the model
test_model(validation, model, device)
print_prediction(model, device)
arch = results.pretrained_model
# Load and preprocess data
train_loader, test_loader, validate_loader, train_data, test_data, validate_data = load_data(
data_dir)
# Load pretrained model
pre_train_model = results.pretrained_model
model = getattr(models, pre_train_model)(pretrained=True)
# Build and attach new classifier
input_unit_count = model.classifier[0].in_features
build_classifier(model, input_unit_count, hidden_unit_count, dropout_rate)
# Using a NLLLoss as output is LogSoftmax
criterion = nn.NLLLoss()
# Using Adam optimiser algorithm - uses concept of momentum to add fractions
# to previous gradient descents to minimize loss function
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
# Train model
model, optimizer = train_model(model, epochs, train_loader, validate_loader,
criterion, optimizer, gpu_mode)
# Test model
test_model(model, test_loader, gpu_mode)
# Save model
save_model(model, pre_train_model, train_data, optimizer, save_dir, epochs)
action='store',
type=int,
help='Choose number of hidden epochs')
parser.add_argument('gpu',
action='store',
help='Use GPU',
choices=['GPU', 'CPU'])
args = parser.parse_args()
#Save user's inputs to use them as parameters
data_directory = args.data_directory
save_directory = args.save_dir
model_arch = args.arch
learning_rate = args.learning_rate
hidden_units = args.hidden_units
epochs = args.epochs
gpu = args.gpu
if gpu == 'GPU':
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
else:
device = "cpu"
train_data, trainloader, validloader, testloader = utility_functions.loading_data(
data_directory)
model, criterion, optimizer = functions.Network(model_arch, hidden_units,
learning_rate)
functions.train_model(epochs, model, trainloader, criterion, optimizer,
validloader, gpu)
functions.save_model(model, epochs, hidden_units, optimizer, train_data,
save_directory)
from functions import train_model
# train final model for each shot subtype using hyperparameter id_num:
train_model('direct', 2)
train_model('head_cross', 7)
train_model('cross', 9)
train_model('head', 16)
train_model('regular', 17)
type=bool,
default=True)
args = parser.parse_args()
device = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu') if args.gpu else 'cpu'
data_dir = args.data_directory
save_dir = args.save_directory
learning_rate = args.lr
hidden_units = args.units
epochs = args.num_epochs
arch = args.model_arch
# Process and load the data
train_data, vaild_data, test_data, trainloader, validloader, testloader = load_data(
data_dir)
# Loading the pre-trained network
model = getattr(models, arch)(pretrained=True)
input_units = model.classifier[0].in_features
# Creating the model
model = ProjectClassifier(model, input_units, hidden_units)
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate)
model.to(device)
#Training the model
model, optimizer = train_model(model, trainloader, validloader, criterion,
optimizer, epochs, device)
# Testing the model
test_model(model, testloader, criterion, device)
# Saving the model
save_model(model, arch, epochs, optimizer, train_data, save_dir)
#X_test = scaler.transform(X_test)
oof_lr, prediction_lr, _ = train_model(X_train_.values, X_test_.values, y_train.values, params=None, model_type='sklearn', model=model, folds = folds, n_fold = n_fold)# 0.9444
'''
# b) sklearn feature selection
'''
selector = SelectKBest(f_classif, k=10)
X_trainK = selector.fit_transform(X_train.values, y_train.values)
X_testK = selector.transform(X_test.values)
oof_lr_1, prediction_lr_1, scores = train_model(X_trainK, X_testK, y_train.values, params=None, model_type='sklearn', model=model, folds = folds, n_fold = n_fold) # 0.9402
'''
# c) SHAP
'''
explainer = shap.LinearExplainer(model, X_train)
shap_values = explainer.shap_values(X_train)
shap.summary_plot(shap_values, X_train)
'''
# d) ELI5
import eli5
oof_lr, prediction_lr, _ = train_model(X_train.values,
X_test.values,
y_train.values,
params=None,
model_type='sklearn',
model=model,
folds=folds,
n_fold=n_fold)
print eli5.show_weights(model, top=10)
image_datasets = {
x: ImageDataset(dataType=x, dataAug=(args['data_aug'] == 1))
for x in ['train', 'val']
}
dataloaders_dict = {
x: DataLoader(image_datasets[x],
batch_size=args['batchsize'],
shuffle=True,
num_workers=args['num_workers'])
for x in ['train', 'val']
}
#choose different loss function
if (args['loss_type'] == 'L1'):
criterions = [nn.CrossEntropyLoss(), nn.L1Loss()]
elif (args['loss_type'] == 'SmoothL1Loss'):
criterions = [nn.CrossEntropyLoss(), nn.SmoothL1Loss()]
else:
criterions = [nn.CrossEntropyLoss(), nn.MSELoss()]
model_ft, hist = train_model(model,
dataloaders_dict,
criterions,
optimizer,
num_epochs=args['num_epochs'],
device=device,
loss_ratio=args['loss_ratio'],
lr_decay=args['lr_decay'])
torch.save(model_ft.state_dict(), args['storeParamName'])
x: ImageDataset(root_dir, data_use=x, transform_list=transform_list)
for x in ['train', 'val']
}
dataloaders_dict = {
x: DataLoader(image_datasets[x],
batch_size=32,
shuffle=True,
num_workers=4)
for x in ['train', 'val']
}
criterions = [nn.CrossEntropyLoss(), nn.MSELoss()]
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [70, 90],
gamma=0.1)
print_out_file = './print_output.txt'
model_ft, val_acc_history = train_model(model,
dataloaders_dict,
criterions,
optimizer,
scheduler,
device=device,
num_epochs=100,
loss_ratio=0.5,
print_file=print_out_file)
PATH = './model.pt'
torch.save(model_ft.state_dict(), PATH)
def main(argv):
parser = argparse.ArgumentParser(
description='Train model to predict flower name from an image.',
prog='train.py',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('data_dir',
type=dir_path,
nargs=1,
help='Path to the directory containing the images.')
parser.add_argument('--save_dir',
type=dir_path,
nargs=1,
default=['.'],
help='Set directory to save checkpoints.')
parser.add_argument('--arch',
type=str,
nargs=1,
default=['vgg16'],
help='Architecture.',
choices=['vgg11', 'vgg13', 'vgg16', 'vgg19'])
parser.add_argument('--learning_rate',
type=float,
nargs=1,
default=[0.001],
help='Learning rate.')
parser.add_argument('--hidden_units',
type=int,
nargs=1,
default=[512],
help='Hidden units.')
parser.add_argument('--epochs',
type=int,
nargs=1,
default=[20],
help='Number of epochs.')
parser.add_argument('--gpu',
nargs='?',
const='cuda',
help='Use GPU for inference.')
args = parser.parse_args(argv)
vargs = vars(args)
data_dir = vargs['data_dir'][0]
save_dir = vargs['save_dir'][0]
arch = vargs['arch'][0]
learning_rate = vargs['learning_rate'][0]
epochs = vargs['epochs'][0]
if epochs < 1:
print('epochs must be an integer greater or equal to 1')
sys.exit(2)
hidden_units = vargs['hidden_units'][0]
if hidden_units < 102:
print('hidden_units must be an integer greater or equal to 102')
sys.exit(2)
device = vargs['gpu']
if device == None:
device = 'cpu'
print('parameters used for training:')
print('-----------------------------')
print("data_dir = {}".format(data_dir))
print("save_dir={}".format(save_dir))
print("arch={}".format(arch))
print("learning_rate={}".format(learning_rate))
print("hidden_units={}".format(hidden_units))
print("epochs={}".format(epochs))
print("device={}".format(device))
train_dir = data_dir + '/train'
if not os.path.isdir(train_dir):
print('{} does not contain the mandatory train subdirectory'.format(
data_dir))
sys.exit(2)
valid_dir = data_dir + '/valid'
if not os.path.isdir(valid_dir):
print('{} does not contain the mandatory valid subdirectory'.format(
data_dir))
sys.exit(2)
test_dir = data_dir + '/test'
if not os.path.isdir(test_dir):
print('{} does not contain the mandatory test subdirectory'.format(
data_dir))
sys.exit(2)
# TODO: Define your transforms for the training, validation, and testing sets
data_transforms = {
'train':
transforms.Compose([
transforms.RandomRotation(45),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'valid':
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'test':
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
}
# TODO: Load the datasets with ImageFolder
image_datasets = {
data_type: datasets.ImageFolder(os.path.join(data_dir, data_type),
data_transforms[data_type])
for data_type in ['train', 'valid', 'test']
}
# TODO: Using the image datasets and the trainforms, define the dataloaders
dataloaders = {
data_type: torch.utils.data.DataLoader(image_datasets[data_type],
batch_size=32,
shuffle=True)
for data_type in ['train', 'valid', 'test']
}
dataset_sizes = {
data_type: len(image_datasets[data_type])
for data_type in ['train', 'valid', 'test']
}
# TODO: Build and train your network
if arch == 'vgg11':
model = models.vgg11(pretrained=True)
elif arch == 'vgg13':
model = models.vgg13(pretrained=True)
elif arch == 'vgg16':
model = models.vgg16(pretrained=True)
elif arch == 'vgg19':
model = models.vgg19(pretrained=True)
else:
print('Architecture {} not supported'.format(arch))
print('Supported architectures are vgg11, vgg13, vgg16 and vgg19')
sys.exit(2)
for param in model.parameters():
param.requires_grad = False
classifier = nn.Sequential(
OrderedDict([('fc1', nn.Linear(25088, hidden_units)),
('relu1', nn.ReLU()), ('dropout1', nn.Dropout(0.2)),
('fc2', nn.Linear(hidden_units, hidden_units)),
('relu2', nn.ReLU()), ('dropout2', nn.Dropout(0.2)),
('fc3', nn.Linear(hidden_units, 102)),
('output', nn.LogSoftmax(dim=1))]))
model.classifier = classifier
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate)
scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
functions.train_model(model, criterion, optimizer, scheduler, epochs,
device, dataloaders, dataset_sizes)
functions.test_model(model, dataloaders, device=device)
# TODO: Save the checkpoint
model.class_to_idx = image_datasets['train'].class_to_idx
#always saving on cpu to reuse without having to consume gpu time
model.to('cpu')
checkpoint2 = {
'network': arch,
'classifier': model.classifier,
'state_dict': model.state_dict(),
'class_to_idx': model.class_to_idx
}
torch.save(checkpoint2, save_dir + '/checkpoint2.pth')
def feature_extract():
'''
1. Load data
torchvision 라이브러리의 transform을 통해서 훈련 데이터에 대해서 Crop, augmentation, Normalize를,
Validation 데이터에 대해서 Crop, Normalize를 편리하게 수행할 수 있습니다.
'''
data_transforms = {
'train':
transforms.Compose([
transforms.RandomSizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'val':
transforms.Compose([
transforms.Scale(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
# 데이터 경로 및 data loader 설정
data_dir = 'data'
dsets = {
x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])
for x in ['train', 'val']
}
dset_loaders = {
x: torch.utils.data.DataLoader(dsets[x],
batch_size=4,
shuffle=True,
num_workers=4)
for x in ['train', 'val']
}
# 데이터 사이즈와 클래스
dset_sizes = {x: len(dsets[x]) for x in ['train', 'val']}
dset_classes = dsets['train'].classes
# GPU 사용 가능 여부
use_gpu = torch.cuda.is_available()
# Data Loader에서 batch size 만큼의 훈련 데이터 및 클래스 로드
inputs, classes = next(iter(dset_loaders['train']))
# 한 batch 안의 이미지들을 grid로 만들어서 확인해 보기
out = torchvision.utils.make_grid(inputs)
imshow(out, title=[dset_classes[x] for x in classes])
'''
2. Model Load and Modify
'''
# Pretrain된 18 layer residual network를 로드
model_conv = torchvision.models.resnet18()
model_conv.load_state_dict(
torch.load('./data/models/resnet18-5c106cde.pth'))
# Load한 모델의 파라미터를 train 과정에서 변경하지 않도록 설정 (requires_grad = False)
for param in model_conv.parameters():
param.requires_grad = False
# 마지막 레이어를 타겟 task에 맞게 수정
print("Original Fully connected layer of resnet18 (Last layer):",
model_conv.fc)
num_ftrs = model_conv.fc.in_features
model_conv.fc = nn.Linear(num_ftrs, 2)
print("Modified Fully connected layer of resnet18 (Last layer):",
model_conv.fc)
if use_gpu:
model_conv = model_conv.cuda()
# Loss function 설정
criterion = nn.CrossEntropyLoss()
# 모델의 Optimizer 설정
optimizer_conv = optim.SGD(model_conv.fc.parameters(),
lr=0.001,
momentum=0.9)
'''
3. Train & Evaluate
'''
model_conv = train_model(model_conv,
criterion,
optimizer_conv,
exp_lr_scheduler,
use_gpu,
dset_loaders,
dset_sizes,
num_epochs=25)
'''
4. model visualize
'''
visualize_model(model_conv, dset_loaders, dset_classes, use_gpu)
plt.ioff()
plt.show()
return
def handle_command(command):
global historicalData, sizeOfModelData, marketData, symbolData, periodData, model, n_per_in, n_per_out, n_features, modelWeights
global modelWeights, percentTestData, epochs, batch_size, earlyStop, saveWeights, displayResults, df, close_scaler, testDf, numberToPredict
global percentToPredict, predictions, shortDf, actual, normalizedDf, trainDf, normalizedTrainDf, startAmount, tradingFee, train_close_scaler, calcPredictionsNumberPredicts
global calcPredictions, calcShortDf, prevPredictions, binaryModel
if (command == "exit"):
#Add other exit routines
return "exit"
elif (command == "help"):
print_commands()
return None
elif ("loaddata" in command):
if ("-file" in command):
file = grabValue(command, "-file")
historicalData = file
if ("-size" in command):
size = grabValue(command, "-size")
sizeOfModelData = size
if ("-percentTestData" in command):
percentTestData = float(grabValue(command, "-percentTestData"))
if ("-market" in command):
market = grabValue(command, "-market")
marketData = market
if ("-symbol" in command):
symbol = grabValue(command, "-symbol")
symbolData = symbol
if ("-period" in command):
period = grabValue(command, "-period")
periodData = period
df, testDf, trainDf = functions.get_data(
periodData,
marketData,
symbolData,
percentTestData=percentTestData,
saveData=False,
historical=historicalData,
size=sizeOfModelData)
normalizedTrainDf, train_close_scaler = functions.normalize_df(trainDf)
normalizedDf, close_scaler = functions.normalize_df(df)
elif ("trainmodel" in command):
loadWeights = False
pathToWeights = modelWeights
binary = False
if ("-evaluate" in command):
testDf.to_csv("Test df xasd.csv")
normalizedTestDf, test_close_scaler = functions.normalize_df(
testDf)
X_test, y_test = functions.split_sequence_binary(
normalizedTestDf.to_numpy(), n_per_in, df, test_close_scaler)
scores = binaryModel.evaluate(X_test, y_test, verbose=1)
print(scores)
return
if ("-loadmodel" in command):
file = grabValue(command, "-loadmodel")
loadWeights = True
pathToWeights = file
if ("-epochs" in command):
epochs = int(grabValue(command, "-epochs"))
if ("-batch_size" in command):
batch_size = int(grabValue(command, "-batch_size"))
if ("-earlystop" in command):
earlyStop = grabValue(command, "-earlystop")
if ("-saveWeights" in command):
saveWeights = grabValue(command, "-saveWeights")
if ("-displayResults" in command):
displayResults = grabValue(command, "-displayResults")
if ("-binary" in command):
binary = True
if (binary):
functions.train_model_binary(normalizedTrainDf,
epochs,
batch_size,
binaryModel,
n_per_in,
n_per_out,
loadWeights=loadWeights,
pathToWeights=pathToWeights,
earlyStop=earlyStop,
saveWeights=saveWeights,
displayResults=displayResults,
close_scaler=train_close_scaler)
else:
functions.train_model(normalizedTrainDf,
epochs,
batch_size,
model,
n_per_in,
n_per_out,
loadWeights=loadWeights,
pathToWeights=pathToWeights,
earlyStop=earlyStop,
saveWeights=saveWeights,
displayResults=displayResults,
close_scaler=train_close_scaler)
elif ("plotpredictions" in command):
if (len(predictions) == 0):
predictions, shortDf, prevPredictions = functions.get_predictions(
normalizedDf,
n_per_in,
n_per_out,
n_features,
close_scaler,
model,
False,
numberPredictions=numberToPredict)
actual = functions.transform_back(normalizedDf, close_scaler)
if ("-prev" in command):
functions.plot_predictions_actual_prev(predictions, actual,
prevPredictions)
else:
functions.plot_predictions_actual(predictions, actual)
elif ("predict" in command):
timeRemaining = True
binary = False
if ("-timeRemaining" in command):
timeRemaining = grabValue(command, "-timeRemaining")
if ("-binary" in command):
binary = True
if ("-numberToPredict" in command):
numberToPredict = int(grabValue(command, "-numberToPredict"))
if ("-percentToPredict" in command):
percentToPredict = float(grabValue(command, "-percentToPredict"))
numberToPredict = (int(percentToPredict * len(df)))
if (binary):
predictions, shortDf, prevPredictions = functions.get_predictions_binary(
normalizedDf,
n_per_in,
n_per_out,
n_features,
close_scaler,
model,
timeRemaining,
numberPredictions=numberToPredict)
else:
predictions, shortDf, prevPredictions = functions.get_predictions(
normalizedDf,
n_per_in,
n_per_out,
n_features,
close_scaler,
model,
timeRemaining,
numberPredictions=numberToPredict)
elif ("marketprice" in command):
Tdf, TtestDf, TtrainDf = functions.get_data(periodData,
marketData,
symbolData,
percentTestData=0.0,
saveData=False,
historical=historicalData,
size=sizeOfModelData)
TnormalizedTrainDf, _ = functions.normalize_df(TtrainDf)
TnormalizedDf, Tclose_scaler = functions.normalize_df(Tdf)
smallPredictions, smallShortDf, _ = functions.get_predictions(
TnormalizedDf,
n_per_in,
n_per_out,
n_features,
Tclose_scaler,
model,
False,
numberPredictions=n_per_in + 5)
if ("-current" in command):
print("Current real price as of: " + str(df.tail(1).index[0]) +
" : " + str(df.tail(1).Close[0]))
print("Current predicted price as of: " +
str(smallPredictions.tail(1).index[0]) + " : " +
str(smallPredictions.tail(1).Close[0]))
if ("-next" in command):
nextPrice, prevPrice = functions.get_next_predicted_price(
normalizedDf, n_per_in, n_features, model, close_scaler)
print("Current predicted price as of: " +
str(smallPredictions.tail(1).index[0]) + " : " +
str(prevPrice))
print("Next predicted price as of: " +
str(smallPredictions.tail(1).index[0] + timedelta(hours=1)) +
" : " + str(nextPrice))
elif ("calcgains" in command):
showInfo = True
noLoss = False
numberHours = int(percentTestData * len(df))
opposite = False
cashout = False
conservebuy = False
if ("-start" in command):
startAmount = int(grabValue(command, "-start"))
if ("-tradingFee" in command):
tradingFee = float(grabValue(command, "-tradingFee"))
if ("-info" in command):
showInfo = grabValue(command, "-info")
if ("-noloss" in command):
noLoss = True
if ("-numberHours" in command):
numberHours = int(grabValue(command, "-numberHours"))
if ("-opposite" in command):
opposite = True
if ("-cashout" in command):
cashout = True
if ("-conservebuy" in command):
conservebuy = True
if (numberHours + n_per_in != calcPredictionsNumberPredicts):
calcPredictionsNumberPredicts = numberHours + n_per_in
calcPredictions, calcShortDf, _ = functions.get_predictions(
normalizedDf,
n_per_in,
n_per_out,
n_features,
close_scaler,
model,
True,
numberPredictions=calcPredictionsNumberPredicts)
actual = functions.transform_back(normalizedDf, close_scaler)
if ("-sequence" in command):
avgList = []
print("Length calcPredictions: " + str(len(calcPredictions)))
print("Length n_per_in: " + str(n_per_in))
print("len(calcPredictions)-n_per_in: " +
str(len(calcPredictions) - n_per_in))
for i in range(len(calcPredictions), n_per_in, -1):
# print("yo")
endAmount = functions.calc_gain(actual,
calcPredictions.tail(i),
startAmount,
tradingFee,
n_per_in,
n_per_out,
noLoss,
cashout,
opposite,
showInfo=False)
# print(endAmount)
gain = endAmount - startAmount
gainPerHour = gain / (i - n_per_in)
print("Gain per hour iteration(" + str(i - n_per_in) + ") : " +
str(gainPerHour))
avgList.append(gainPerHour)
print("\n\nFinal Sequence Statistics: \n\n")
print("Starting dollar amount: " + str(startAmount) + "\n")
startBTC = float(
actual.tail(len(calcPredictions))["Close"][n_per_in - 1])
endBTC = float(actual.tail(1)["Close"][0])
avgBTCChange = (
(endBTC - startBTC) /
(len(calcPredictions) - n_per_in)) * (startAmount / startBTC)
print("Avg dollar gain per hour: " + str(mean(avgList)))
print("Avg BTC change per hour(dollars): " + str(avgBTCChange))
print()
endAmountAvg = (mean(avgList) *
(len(calcPredictions) - n_per_in)) + startAmount
endBTCHeld = (
(endBTC - startBTC) / startBTC) * startAmount + startAmount
print("Ending dollar amount traded (average): " +
str(endAmountAvg))
print("Ending dollar amount if invested/held: " + str(endBTCHeld))
print()
percentChangeBTC = (((endBTC - startBTC) / startBTC) * 100)
percentChangePort = (((endAmountAvg - startAmount) / startAmount) *
100)
print("Percent change in BTC: " + str(round(percentChangeBTC, 2)) +
"%")
print("Percent change in Portfolio: " +
str(round(percentChangePort, 2)) + "%")
print()
maxGainHour = max(avgList)
indexMax = len(calcPredictions) - avgList.index(
maxGainHour) - n_per_in
print("Highest Gain/Hour: " + str(maxGainHour))
print("Highest Gain From Hour: " + str(indexMax))
print()
else:
functions.calc_gain(actual,
calcPredictions,
startAmount,
tradingFee,
n_per_in,
n_per_out,
noLoss,
cashout,
conservebuy,
opposite,
showInfo=showInfo)
elif ("printmodel" in command):
print()
model.summary()
print()
return None
# Set device(gpu or cpu)
if results.gpu == True:
device = 'cuda'
else:
device = 'cpu'
# Load data
train_loader, validation_loader, test_loader, train_data, validation_data, test_data = load_data(
data_dir)
# Load pretrained model
model = getattr(models, pre_train_model)(pretrained=True)
# Build new classifier
input_units = model.classifier[0].in_features
construct_classifier(model, input_units, hidden_units, dropout)
# Define criterion and optimizer
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
# Train model
model, optimizer = train_model(model, epochs, train_loader, validation_loader,
criterion, optimizer, device)
# Test model
test_model(model, test_loader, criterion, device)
# Save model
save_model(model, save_dir, epochs, optimizer, train_data)
dropout = results.dropout
hidden_units = results.hunits
epochs = results.number_epochs
gpu = results.gpu
# Load Data
train_data, valid_data, test_data, trainloader, validloader, testloader = load_data(data_dir)
#model = models.pt_model(pretrained = True)
model = getattr(models,pt_model)(pretrained = True)
#loaded_model = load_checkpoint(model, save_dir)
# Build classifier
input_units = model.classifier[0].in_features
build_classifier(model, input_units, hidden_units, dropout)
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), learning_rate)
# Train model
model, optimizer = train_model(model, epochs, trainloader, validloader, criterion, optimizer, gpu)
# Test model
test_model(model, testloader, gpu)
# Save model
save_model(model, train_data, optimizer, save_dir, epochs)
loss_ = loss_from_config
print "\nLoss = " + str(loss_)
model.compile(loss=loss_,
optimizer=user_defined_optimizer,
metrics=[config.get('KERAS', 'metrics')])
d = model.summary()
#----------------------------------------------------------------------------------------------------
# Training or Reading model
if config.getboolean('KERAS', 'train'):
# Train the network
history = fcn.train_model(model,
X_train,
Y_train,
features,
cb_list,
config,
sample_weights=sample_weights_)
#----------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------
#### Predict and visualize results ####
#######################################
# Prediction
#-----------
reload(fcn)
df_pred = fcn.make_prediction_higgs(model, X_test, Y_test, features, config)
#----------------------------------------------------------------------------------------------------
# Visualization
#--------------
reload(sig)
