def main(): start_time = time.time() in_arg = args_input() # get the data dataset, dataloaders = functions.load_data() # get the model model = functions.get_model(in_arg.arch) # get the classifier, criterion, optimizer, device model, classifier, criterion, optimizer, device = network_param( model, in_arg.arch, in_arg.hidden_units, in_arg.learning_rate, in_arg.gpu) model.to(device) print('Training model...') functions.train_model(model, criterion, optimizer, device, dataloaders['train'], dataloaders['val'], in_arg.epochs) # validation on test data print('\nGetting results on test data accuracy...') functions.test_model(model, criterion, device, dataloaders['test']) # saving checkpoint on trained model print('\nSaving checkpoint for current trained model...') functions.save_checkpoint(model, optimizer, dataset, in_arg.arch, in_arg.epochs, in_arg.save_dir) print('Checkpoint saved!')
def main(): parser = argparse.ArgumentParser( description='This program predicts a flower name from an image') parser.add_argument('data_dir', type=str, help='Dataset directory') parser.add_argument('--save_dir', type=str, default='./', help='Saved checkpoint directory') parser.add_argument('--arch', type=str, default='vgg16', help='Network architecture') parser.add_argument('--hidden_units', type=int, default='256', help='Hidden units') parser.add_argument('--dropout', type=float, default='0.2', help='Dropout for the hidden layers') parser.add_argument('--num_classes', type=int, default='256', help='Number of classes for classification') parser.add_argument('--learning_rate', type=float, default='0.005', help='Learning rate') parser.add_argument('--epochs', type=int, default='20', help='Number of epochs') parser.add_argument('--gpu', action='store_true', help='Use GPU', default=False) args = parser.parse_args() device = torch.device("cuda:0" if torch.cuda.is_available() else 'cpu' ) if args.gpu else 'cpu' model, criterion, optimizer, scheduler = initialize_model( args.arch, args.hidden_units, args.dropout, args.num_classes, device, args.learning_rate) dataloaders, image_datasets = create_loaders(args.data_dir) train_model(model, dataloaders, criterion, optimizer, scheduler, image_datasets, args.epochs, device) test_model(model, dataloaders, image_datasets, device) save_checkpoint(model, optimizer, scheduler, args.epochs, args.learning_rate, f'{args.arch}_checkpoint.pth')
def full_execution(midi_files, output_tag, total_epochs, batch_size, sequence_length, temperature, offset_adj): # epoch_stops = 1 # epoch_count = 0 weight_file = None note_file = fxn.convert_midis_to_notes(midi_files, output_tag) epochs = total_epochs with open(note_file, 'rb') as filepath: notes = pickle.load(filepath) network_input, network_output, n_patterns, n_vocab, pitchnames = fxn.prepare_sequences( notes, sequence_length) network_input_r, network_output_r = fxn.reshape_for_training( network_input, network_output, sequence_length) # while epoch_count <= total_epochs: # epochs = epoch_stops # model = fxn.create_network(network_input_r, n_vocab, weight_file) model, weight_file, history = fxn.train_model(model, network_input_r, network_output_r, epochs, batch_size, output_tag, sequence_length) normalized_input = fxn.reshape_for_creation(network_input, n_patterns, sequence_length, n_vocab) model = fxn.create_network(normalized_input, n_vocab, weight_file) prediction_output = fxn.generate_notes(model, network_input, pitchnames, sequence_length, notes_generated, n_vocab, temperature) output_notes = fxn.create_midi(prediction_output, output_tag, sequence_length, offset_adj) # epoch_count += epoch_stops return output_notes, history, weight_file
def main(): start_time = time() in_arg = get_input_args() #check_command_line_arguments(in_arg) images_datasets = data_loaders(in_arg.data_dir) if in_arg.gpu is True: if torch.cuda.is_available(): device = torch.device('cuda') else: device = torch.device('cpu') print("Using CPU since GPU is not available") else: device = torch.device('cpu') model = getattr(models, in_arg.arch)(pretrained=True) for param in model.features.parameters(): param.require_grad = False output_classes = 102 hidden_units = 2000 if in_arg.arch == 'vgg16': classifier = nn.Sequential( OrderedDict([('fc1', nn.Linear(25088, 4096)), ('relu', nn.ReLU()), ('dropout', nn.Dropout(p=0.5)), ('fc2', nn.Linear(4096, in_arg.hidden_units)), ('relu', nn.ReLU()), ('dropout', nn.Dropout(p=0.5)), ('fc3', nn.Linear(in_arg.hidden_units, output_classes))])) model.classifier = classifier if in_arg.arch == 'alexnet': classifier = nn.Sequential( OrderedDict([('dropout', nn.Dropout(p=0.5)), ('fc1', nn.Linear(9216, 4096)), ('relu', nn.ReLU()), ('dropout', nn.Dropout(p=0.5)), ('fc2', nn.Linear(4096, in_arg.hidden_units)), ('relu', nn.ReLU()), ('fc3', nn.Linear(in_arg.hidden_units, output_classes))])) model.classifier = classifier model = model.to(device) criterion = nn.CrossEntropyLoss() l_r = in_arg.learning_rate epochs = in_arg.epochs optimizer = optim.SGD(model.classifier.parameters(), lr=l_r, momentum=0.9) exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1) model = train_model(model, criterion, optimizer, images_datasets, exp_lr_scheduler, device, epochs) model.class_to_idx = images_datasets['train'].class_to_idx checkpoint = { 'class_to_idx': model.class_to_idx, 'arch': in_arg.arch, 'classifier': model.classifier, 'state_dict': model.state_dict(), 'epochs': in_arg.epochs, 'learning_rate': in_arg.learning_rate, 'hidden_units': in_arg.hidden_units } torch.save(checkpoint, in_arg.save_dir) end_time = time()
def main(): args = parser_fun_train() with open('cat_to_name.json', 'r') as f: cat_to_name = json.load(f) #training the model optimizer, model = train_model(args.arch, imd.train_dl, imd.valid_dl, args.epochs, 40, args.device, args.hidden_units, args.lr) create_checkpoint(model, '', args.arch, imd.training_dataset.class_to_idx, optimizer, args.epochs)
def train_model_launcher(r, w, tP, u_ed, xi, save_results=False, i=''): bayes_opt_score = functions.train_model( r, w, tP, u_ed, xi, save_results=save_results, i=i, config_parameters=config_parameters, fileName=fileName) return bayes_opt_score
def main(): #print("hello world") for luck in_arg = train_input_args() print(" data directory =",in_arg.data_dir, "\n save directory =", in_arg.save_dir, "\n model architecture =", in_arg.arch, "\n hidden units =", in_arg.hidden_units, "\n learning rate =", in_arg.learning_rate, "\n epochs =", in_arg.epochs, "\n device =", in_arg.device) image_data, data_loader = load_images() model, optimizer, criterion = build_model(in_arg.arch , in_arg.hidden_units, in_arg.learning_rate) model = train_model(model,data_loader, in_arg.epochs, criterion, optimizer, in_arg.device) save_model(model, optimizer, in_arg.arch, in_arg.data_dir, in_arg.save_dir,image_data) print("-"*40)
def main(): # Get CLI arguments args = get_input_args() # Prep data train_transform = utilities.transform_data('train') test_transform = utilities.transform_data('test') # Dataloaders trainloader = utilities.load_data(args.data_directory + '/' + 'train', train_transform) validationloader = utilities.load_data(args.data_directory + '/' + 'valid', test_transform) # Setup and train model model, optimizer, criterion = functions.model_setup( args.arch, args.hidden_units, args.learning_rate) trained_model = functions.train_model(optimizer, criterion, model, trainloader, validationloader, args.gpu, args.epochs) # Save the model functions.save_checkpoint(trained_model, args.save_dir)
import torch from model import Net from functions import load_data, train_model, test_model, print_prediction lr = 0.001 if __name__ == '__main__': train, validation = load_data() cuda = torch.cuda.is_available() device = torch.device("cuda" if cuda else "cpu") model = Net().to(device) optimizer = torch.optim.Adam(model.parameters(), lr=lr) # train the model train_model(train, optimizer, model, device) # test the model test_model(validation, model, device) print_prediction(model, device)
arch = results.pretrained_model # Load and preprocess data train_loader, test_loader, validate_loader, train_data, test_data, validate_data = load_data( data_dir) # Load pretrained model pre_train_model = results.pretrained_model model = getattr(models, pre_train_model)(pretrained=True) # Build and attach new classifier input_unit_count = model.classifier[0].in_features build_classifier(model, input_unit_count, hidden_unit_count, dropout_rate) # Using a NLLLoss as output is LogSoftmax criterion = nn.NLLLoss() # Using Adam optimiser algorithm - uses concept of momentum to add fractions # to previous gradient descents to minimize loss function optimizer = optim.Adam(model.classifier.parameters(), learning_rate) # Train model model, optimizer = train_model(model, epochs, train_loader, validate_loader, criterion, optimizer, gpu_mode) # Test model test_model(model, test_loader, gpu_mode) # Save model save_model(model, pre_train_model, train_data, optimizer, save_dir, epochs)
action='store', type=int, help='Choose number of hidden epochs') parser.add_argument('gpu', action='store', help='Use GPU', choices=['GPU', 'CPU']) args = parser.parse_args() #Save user's inputs to use them as parameters data_directory = args.data_directory save_directory = args.save_dir model_arch = args.arch learning_rate = args.learning_rate hidden_units = args.hidden_units epochs = args.epochs gpu = args.gpu if gpu == 'GPU': device = torch.device("cuda" if torch.cuda.is_available() else "cpu") else: device = "cpu" train_data, trainloader, validloader, testloader = utility_functions.loading_data( data_directory) model, criterion, optimizer = functions.Network(model_arch, hidden_units, learning_rate) functions.train_model(epochs, model, trainloader, criterion, optimizer, validloader, gpu) functions.save_model(model, epochs, hidden_units, optimizer, train_data, save_directory)
from functions import train_model # train final model for each shot subtype using hyperparameter id_num: train_model('direct', 2) train_model('head_cross', 7) train_model('cross', 9) train_model('head', 16) train_model('regular', 17)
type=bool, default=True) args = parser.parse_args() device = torch.device( 'cuda:0' if torch.cuda.is_available() else 'cpu') if args.gpu else 'cpu' data_dir = args.data_directory save_dir = args.save_directory learning_rate = args.lr hidden_units = args.units epochs = args.num_epochs arch = args.model_arch # Process and load the data train_data, vaild_data, test_data, trainloader, validloader, testloader = load_data( data_dir) # Loading the pre-trained network model = getattr(models, arch)(pretrained=True) input_units = model.classifier[0].in_features # Creating the model model = ProjectClassifier(model, input_units, hidden_units) criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate) model.to(device) #Training the model model, optimizer = train_model(model, trainloader, validloader, criterion, optimizer, epochs, device) # Testing the model test_model(model, testloader, criterion, device) # Saving the model save_model(model, arch, epochs, optimizer, train_data, save_dir)
#X_test = scaler.transform(X_test) oof_lr, prediction_lr, _ = train_model(X_train_.values, X_test_.values, y_train.values, params=None, model_type='sklearn', model=model, folds = folds, n_fold = n_fold)# 0.9444 ''' # b) sklearn feature selection ''' selector = SelectKBest(f_classif, k=10) X_trainK = selector.fit_transform(X_train.values, y_train.values) X_testK = selector.transform(X_test.values) oof_lr_1, prediction_lr_1, scores = train_model(X_trainK, X_testK, y_train.values, params=None, model_type='sklearn', model=model, folds = folds, n_fold = n_fold) # 0.9402 ''' # c) SHAP ''' explainer = shap.LinearExplainer(model, X_train) shap_values = explainer.shap_values(X_train) shap.summary_plot(shap_values, X_train) ''' # d) ELI5 import eli5 oof_lr, prediction_lr, _ = train_model(X_train.values, X_test.values, y_train.values, params=None, model_type='sklearn', model=model, folds=folds, n_fold=n_fold) print eli5.show_weights(model, top=10)
image_datasets = { x: ImageDataset(dataType=x, dataAug=(args['data_aug'] == 1)) for x in ['train', 'val'] } dataloaders_dict = { x: DataLoader(image_datasets[x], batch_size=args['batchsize'], shuffle=True, num_workers=args['num_workers']) for x in ['train', 'val'] } #choose different loss function if (args['loss_type'] == 'L1'): criterions = [nn.CrossEntropyLoss(), nn.L1Loss()] elif (args['loss_type'] == 'SmoothL1Loss'): criterions = [nn.CrossEntropyLoss(), nn.SmoothL1Loss()] else: criterions = [nn.CrossEntropyLoss(), nn.MSELoss()] model_ft, hist = train_model(model, dataloaders_dict, criterions, optimizer, num_epochs=args['num_epochs'], device=device, loss_ratio=args['loss_ratio'], lr_decay=args['lr_decay']) torch.save(model_ft.state_dict(), args['storeParamName'])
x: ImageDataset(root_dir, data_use=x, transform_list=transform_list) for x in ['train', 'val'] } dataloaders_dict = { x: DataLoader(image_datasets[x], batch_size=32, shuffle=True, num_workers=4) for x in ['train', 'val'] } criterions = [nn.CrossEntropyLoss(), nn.MSELoss()] scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [70, 90], gamma=0.1) print_out_file = './print_output.txt' model_ft, val_acc_history = train_model(model, dataloaders_dict, criterions, optimizer, scheduler, device=device, num_epochs=100, loss_ratio=0.5, print_file=print_out_file) PATH = './model.pt' torch.save(model_ft.state_dict(), PATH)
def main(argv): parser = argparse.ArgumentParser( description='Train model to predict flower name from an image.', prog='train.py', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('data_dir', type=dir_path, nargs=1, help='Path to the directory containing the images.') parser.add_argument('--save_dir', type=dir_path, nargs=1, default=['.'], help='Set directory to save checkpoints.') parser.add_argument('--arch', type=str, nargs=1, default=['vgg16'], help='Architecture.', choices=['vgg11', 'vgg13', 'vgg16', 'vgg19']) parser.add_argument('--learning_rate', type=float, nargs=1, default=[0.001], help='Learning rate.') parser.add_argument('--hidden_units', type=int, nargs=1, default=[512], help='Hidden units.') parser.add_argument('--epochs', type=int, nargs=1, default=[20], help='Number of epochs.') parser.add_argument('--gpu', nargs='?', const='cuda', help='Use GPU for inference.') args = parser.parse_args(argv) vargs = vars(args) data_dir = vargs['data_dir'][0] save_dir = vargs['save_dir'][0] arch = vargs['arch'][0] learning_rate = vargs['learning_rate'][0] epochs = vargs['epochs'][0] if epochs < 1: print('epochs must be an integer greater or equal to 1') sys.exit(2) hidden_units = vargs['hidden_units'][0] if hidden_units < 102: print('hidden_units must be an integer greater or equal to 102') sys.exit(2) device = vargs['gpu'] if device == None: device = 'cpu' print('parameters used for training:') print('-----------------------------') print("data_dir = {}".format(data_dir)) print("save_dir={}".format(save_dir)) print("arch={}".format(arch)) print("learning_rate={}".format(learning_rate)) print("hidden_units={}".format(hidden_units)) print("epochs={}".format(epochs)) print("device={}".format(device)) train_dir = data_dir + '/train' if not os.path.isdir(train_dir): print('{} does not contain the mandatory train subdirectory'.format( data_dir)) sys.exit(2) valid_dir = data_dir + '/valid' if not os.path.isdir(valid_dir): print('{} does not contain the mandatory valid subdirectory'.format( data_dir)) sys.exit(2) test_dir = data_dir + '/test' if not os.path.isdir(test_dir): print('{} does not contain the mandatory test subdirectory'.format( data_dir)) sys.exit(2) # TODO: Define your transforms for the training, validation, and testing sets data_transforms = { 'train': transforms.Compose([ transforms.RandomRotation(45), transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'valid': transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'test': transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) } # TODO: Load the datasets with ImageFolder image_datasets = { data_type: datasets.ImageFolder(os.path.join(data_dir, data_type), data_transforms[data_type]) for data_type in ['train', 'valid', 'test'] } # TODO: Using the image datasets and the trainforms, define the dataloaders dataloaders = { data_type: torch.utils.data.DataLoader(image_datasets[data_type], batch_size=32, shuffle=True) for data_type in ['train', 'valid', 'test'] } dataset_sizes = { data_type: len(image_datasets[data_type]) for data_type in ['train', 'valid', 'test'] } # TODO: Build and train your network if arch == 'vgg11': model = models.vgg11(pretrained=True) elif arch == 'vgg13': model = models.vgg13(pretrained=True) elif arch == 'vgg16': model = models.vgg16(pretrained=True) elif arch == 'vgg19': model = models.vgg19(pretrained=True) else: print('Architecture {} not supported'.format(arch)) print('Supported architectures are vgg11, vgg13, vgg16 and vgg19') sys.exit(2) for param in model.parameters(): param.requires_grad = False classifier = nn.Sequential( OrderedDict([('fc1', nn.Linear(25088, hidden_units)), ('relu1', nn.ReLU()), ('dropout1', nn.Dropout(0.2)), ('fc2', nn.Linear(hidden_units, hidden_units)), ('relu2', nn.ReLU()), ('dropout2', nn.Dropout(0.2)), ('fc3', nn.Linear(hidden_units, 102)), ('output', nn.LogSoftmax(dim=1))])) model.classifier = classifier criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), lr=learning_rate) scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) functions.train_model(model, criterion, optimizer, scheduler, epochs, device, dataloaders, dataset_sizes) functions.test_model(model, dataloaders, device=device) # TODO: Save the checkpoint model.class_to_idx = image_datasets['train'].class_to_idx #always saving on cpu to reuse without having to consume gpu time model.to('cpu') checkpoint2 = { 'network': arch, 'classifier': model.classifier, 'state_dict': model.state_dict(), 'class_to_idx': model.class_to_idx } torch.save(checkpoint2, save_dir + '/checkpoint2.pth')
def feature_extract(): ''' 1. Load data torchvision 라이브러리의 transform을 통해서 훈련 데이터에 대해서 Crop, augmentation, Normalize를, Validation 데이터에 대해서 Crop, Normalize를 편리하게 수행할 수 있습니다. ''' data_transforms = { 'train': transforms.Compose([ transforms.RandomSizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), 'val': transforms.Compose([ transforms.Scale(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]), } # 데이터 경로 및 data loader 설정 data_dir = 'data' dsets = { x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val'] } dset_loaders = { x: torch.utils.data.DataLoader(dsets[x], batch_size=4, shuffle=True, num_workers=4) for x in ['train', 'val'] } # 데이터 사이즈와 클래스 dset_sizes = {x: len(dsets[x]) for x in ['train', 'val']} dset_classes = dsets['train'].classes # GPU 사용 가능 여부 use_gpu = torch.cuda.is_available() # Data Loader에서 batch size 만큼의 훈련 데이터 및 클래스 로드 inputs, classes = next(iter(dset_loaders['train'])) # 한 batch 안의 이미지들을 grid로 만들어서 확인해 보기 out = torchvision.utils.make_grid(inputs) imshow(out, title=[dset_classes[x] for x in classes]) ''' 2. Model Load and Modify ''' # Pretrain된 18 layer residual network를 로드 model_conv = torchvision.models.resnet18() model_conv.load_state_dict( torch.load('./data/models/resnet18-5c106cde.pth')) # Load한 모델의 파라미터를 train 과정에서 변경하지 않도록 설정 (requires_grad = False) for param in model_conv.parameters(): param.requires_grad = False # 마지막 레이어를 타겟 task에 맞게 수정 print("Original Fully connected layer of resnet18 (Last layer):", model_conv.fc) num_ftrs = model_conv.fc.in_features model_conv.fc = nn.Linear(num_ftrs, 2) print("Modified Fully connected layer of resnet18 (Last layer):", model_conv.fc) if use_gpu: model_conv = model_conv.cuda() # Loss function 설정 criterion = nn.CrossEntropyLoss() # 모델의 Optimizer 설정 optimizer_conv = optim.SGD(model_conv.fc.parameters(), lr=0.001, momentum=0.9) ''' 3. Train & Evaluate ''' model_conv = train_model(model_conv, criterion, optimizer_conv, exp_lr_scheduler, use_gpu, dset_loaders, dset_sizes, num_epochs=25) ''' 4. model visualize ''' visualize_model(model_conv, dset_loaders, dset_classes, use_gpu) plt.ioff() plt.show() return
def handle_command(command): global historicalData, sizeOfModelData, marketData, symbolData, periodData, model, n_per_in, n_per_out, n_features, modelWeights global modelWeights, percentTestData, epochs, batch_size, earlyStop, saveWeights, displayResults, df, close_scaler, testDf, numberToPredict global percentToPredict, predictions, shortDf, actual, normalizedDf, trainDf, normalizedTrainDf, startAmount, tradingFee, train_close_scaler, calcPredictionsNumberPredicts global calcPredictions, calcShortDf, prevPredictions, binaryModel if (command == "exit"): #Add other exit routines return "exit" elif (command == "help"): print_commands() return None elif ("loaddata" in command): if ("-file" in command): file = grabValue(command, "-file") historicalData = file if ("-size" in command): size = grabValue(command, "-size") sizeOfModelData = size if ("-percentTestData" in command): percentTestData = float(grabValue(command, "-percentTestData")) if ("-market" in command): market = grabValue(command, "-market") marketData = market if ("-symbol" in command): symbol = grabValue(command, "-symbol") symbolData = symbol if ("-period" in command): period = grabValue(command, "-period") periodData = period df, testDf, trainDf = functions.get_data( periodData, marketData, symbolData, percentTestData=percentTestData, saveData=False, historical=historicalData, size=sizeOfModelData) normalizedTrainDf, train_close_scaler = functions.normalize_df(trainDf) normalizedDf, close_scaler = functions.normalize_df(df) elif ("trainmodel" in command): loadWeights = False pathToWeights = modelWeights binary = False if ("-evaluate" in command): testDf.to_csv("Test df xasd.csv") normalizedTestDf, test_close_scaler = functions.normalize_df( testDf) X_test, y_test = functions.split_sequence_binary( normalizedTestDf.to_numpy(), n_per_in, df, test_close_scaler) scores = binaryModel.evaluate(X_test, y_test, verbose=1) print(scores) return if ("-loadmodel" in command): file = grabValue(command, "-loadmodel") loadWeights = True pathToWeights = file if ("-epochs" in command): epochs = int(grabValue(command, "-epochs")) if ("-batch_size" in command): batch_size = int(grabValue(command, "-batch_size")) if ("-earlystop" in command): earlyStop = grabValue(command, "-earlystop") if ("-saveWeights" in command): saveWeights = grabValue(command, "-saveWeights") if ("-displayResults" in command): displayResults = grabValue(command, "-displayResults") if ("-binary" in command): binary = True if (binary): functions.train_model_binary(normalizedTrainDf, epochs, batch_size, binaryModel, n_per_in, n_per_out, loadWeights=loadWeights, pathToWeights=pathToWeights, earlyStop=earlyStop, saveWeights=saveWeights, displayResults=displayResults, close_scaler=train_close_scaler) else: functions.train_model(normalizedTrainDf, epochs, batch_size, model, n_per_in, n_per_out, loadWeights=loadWeights, pathToWeights=pathToWeights, earlyStop=earlyStop, saveWeights=saveWeights, displayResults=displayResults, close_scaler=train_close_scaler) elif ("plotpredictions" in command): if (len(predictions) == 0): predictions, shortDf, prevPredictions = functions.get_predictions( normalizedDf, n_per_in, n_per_out, n_features, close_scaler, model, False, numberPredictions=numberToPredict) actual = functions.transform_back(normalizedDf, close_scaler) if ("-prev" in command): functions.plot_predictions_actual_prev(predictions, actual, prevPredictions) else: functions.plot_predictions_actual(predictions, actual) elif ("predict" in command): timeRemaining = True binary = False if ("-timeRemaining" in command): timeRemaining = grabValue(command, "-timeRemaining") if ("-binary" in command): binary = True if ("-numberToPredict" in command): numberToPredict = int(grabValue(command, "-numberToPredict")) if ("-percentToPredict" in command): percentToPredict = float(grabValue(command, "-percentToPredict")) numberToPredict = (int(percentToPredict * len(df))) if (binary): predictions, shortDf, prevPredictions = functions.get_predictions_binary( normalizedDf, n_per_in, n_per_out, n_features, close_scaler, model, timeRemaining, numberPredictions=numberToPredict) else: predictions, shortDf, prevPredictions = functions.get_predictions( normalizedDf, n_per_in, n_per_out, n_features, close_scaler, model, timeRemaining, numberPredictions=numberToPredict) elif ("marketprice" in command): Tdf, TtestDf, TtrainDf = functions.get_data(periodData, marketData, symbolData, percentTestData=0.0, saveData=False, historical=historicalData, size=sizeOfModelData) TnormalizedTrainDf, _ = functions.normalize_df(TtrainDf) TnormalizedDf, Tclose_scaler = functions.normalize_df(Tdf) smallPredictions, smallShortDf, _ = functions.get_predictions( TnormalizedDf, n_per_in, n_per_out, n_features, Tclose_scaler, model, False, numberPredictions=n_per_in + 5) if ("-current" in command): print("Current real price as of: " + str(df.tail(1).index[0]) + " : " + str(df.tail(1).Close[0])) print("Current predicted price as of: " + str(smallPredictions.tail(1).index[0]) + " : " + str(smallPredictions.tail(1).Close[0])) if ("-next" in command): nextPrice, prevPrice = functions.get_next_predicted_price( normalizedDf, n_per_in, n_features, model, close_scaler) print("Current predicted price as of: " + str(smallPredictions.tail(1).index[0]) + " : " + str(prevPrice)) print("Next predicted price as of: " + str(smallPredictions.tail(1).index[0] + timedelta(hours=1)) + " : " + str(nextPrice)) elif ("calcgains" in command): showInfo = True noLoss = False numberHours = int(percentTestData * len(df)) opposite = False cashout = False conservebuy = False if ("-start" in command): startAmount = int(grabValue(command, "-start")) if ("-tradingFee" in command): tradingFee = float(grabValue(command, "-tradingFee")) if ("-info" in command): showInfo = grabValue(command, "-info") if ("-noloss" in command): noLoss = True if ("-numberHours" in command): numberHours = int(grabValue(command, "-numberHours")) if ("-opposite" in command): opposite = True if ("-cashout" in command): cashout = True if ("-conservebuy" in command): conservebuy = True if (numberHours + n_per_in != calcPredictionsNumberPredicts): calcPredictionsNumberPredicts = numberHours + n_per_in calcPredictions, calcShortDf, _ = functions.get_predictions( normalizedDf, n_per_in, n_per_out, n_features, close_scaler, model, True, numberPredictions=calcPredictionsNumberPredicts) actual = functions.transform_back(normalizedDf, close_scaler) if ("-sequence" in command): avgList = [] print("Length calcPredictions: " + str(len(calcPredictions))) print("Length n_per_in: " + str(n_per_in)) print("len(calcPredictions)-n_per_in: " + str(len(calcPredictions) - n_per_in)) for i in range(len(calcPredictions), n_per_in, -1): # print("yo") endAmount = functions.calc_gain(actual, calcPredictions.tail(i), startAmount, tradingFee, n_per_in, n_per_out, noLoss, cashout, opposite, showInfo=False) # print(endAmount) gain = endAmount - startAmount gainPerHour = gain / (i - n_per_in) print("Gain per hour iteration(" + str(i - n_per_in) + ") : " + str(gainPerHour)) avgList.append(gainPerHour) print("\n\nFinal Sequence Statistics: \n\n") print("Starting dollar amount: " + str(startAmount) + "\n") startBTC = float( actual.tail(len(calcPredictions))["Close"][n_per_in - 1]) endBTC = float(actual.tail(1)["Close"][0]) avgBTCChange = ( (endBTC - startBTC) / (len(calcPredictions) - n_per_in)) * (startAmount / startBTC) print("Avg dollar gain per hour: " + str(mean(avgList))) print("Avg BTC change per hour(dollars): " + str(avgBTCChange)) print() endAmountAvg = (mean(avgList) * (len(calcPredictions) - n_per_in)) + startAmount endBTCHeld = ( (endBTC - startBTC) / startBTC) * startAmount + startAmount print("Ending dollar amount traded (average): " + str(endAmountAvg)) print("Ending dollar amount if invested/held: " + str(endBTCHeld)) print() percentChangeBTC = (((endBTC - startBTC) / startBTC) * 100) percentChangePort = (((endAmountAvg - startAmount) / startAmount) * 100) print("Percent change in BTC: " + str(round(percentChangeBTC, 2)) + "%") print("Percent change in Portfolio: " + str(round(percentChangePort, 2)) + "%") print() maxGainHour = max(avgList) indexMax = len(calcPredictions) - avgList.index( maxGainHour) - n_per_in print("Highest Gain/Hour: " + str(maxGainHour)) print("Highest Gain From Hour: " + str(indexMax)) print() else: functions.calc_gain(actual, calcPredictions, startAmount, tradingFee, n_per_in, n_per_out, noLoss, cashout, conservebuy, opposite, showInfo=showInfo) elif ("printmodel" in command): print() model.summary() print() return None
# Set device(gpu or cpu) if results.gpu == True: device = 'cuda' else: device = 'cpu' # Load data train_loader, validation_loader, test_loader, train_data, validation_data, test_data = load_data( data_dir) # Load pretrained model model = getattr(models, pre_train_model)(pretrained=True) # Build new classifier input_units = model.classifier[0].in_features construct_classifier(model, input_units, hidden_units, dropout) # Define criterion and optimizer criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), learning_rate) # Train model model, optimizer = train_model(model, epochs, train_loader, validation_loader, criterion, optimizer, device) # Test model test_model(model, test_loader, criterion, device) # Save model save_model(model, save_dir, epochs, optimizer, train_data)
dropout = results.dropout hidden_units = results.hunits epochs = results.number_epochs gpu = results.gpu # Load Data train_data, valid_data, test_data, trainloader, validloader, testloader = load_data(data_dir) #model = models.pt_model(pretrained = True) model = getattr(models,pt_model)(pretrained = True) #loaded_model = load_checkpoint(model, save_dir) # Build classifier input_units = model.classifier[0].in_features build_classifier(model, input_units, hidden_units, dropout) criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), learning_rate) # Train model model, optimizer = train_model(model, epochs, trainloader, validloader, criterion, optimizer, gpu) # Test model test_model(model, testloader, gpu) # Save model save_model(model, train_data, optimizer, save_dir, epochs)
loss_ = loss_from_config print "\nLoss = " + str(loss_) model.compile(loss=loss_, optimizer=user_defined_optimizer, metrics=[config.get('KERAS', 'metrics')]) d = model.summary() #---------------------------------------------------------------------------------------------------- # Training or Reading model if config.getboolean('KERAS', 'train'): # Train the network history = fcn.train_model(model, X_train, Y_train, features, cb_list, config, sample_weights=sample_weights_) #---------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------- #### Predict and visualize results #### ####################################### # Prediction #----------- reload(fcn) df_pred = fcn.make_prediction_higgs(model, X_test, Y_test, features, config) #---------------------------------------------------------------------------------------------------- # Visualization #-------------- reload(sig)