def main(): assert FLAGS.MODE in ('train', 'valid', 'demo') if FLAGS.MODE == 'demo': demo(FLAGS.checkpoint_dir, FLAGS.show_box) elif FLAGS.MODE == 'train': train_model(FLAGS.train_data) elif FLAGS.MODE == 'valid': valid_model(FLAGS.checkpoint_dir, FLAGS.valid_data)
def get_trained_model(train_batches, test_batch, weights_in=None, weights_out=None): model = get_compiled_model() test_data, test_labels = get_data_and_labels(test_batch) train_data, train_labels = get_data_and_labels(train_batches) if weights_in: model.load_weights(weights_in) train_model(model, test_data, test_labels, train_data, train_labels) if weights_out: model.save_weights(weights_out) return model
def main(args): # process input file input_file = util.ensure_local_file(args['train_files'][0]) user_map, item_map, tr_sparse, test_sparse = model.create_test_and_train_sets( args, input_file, args['data_type']) # train model output_row, output_col = model.train_model(args, tr_sparse) # save trained model to job directory model.save_model(args, user_map, item_map, output_row, output_col) # log results train_rmse = wals.get_rmse(output_row, output_col, tr_sparse) test_rmse = wals.get_rmse(output_row, output_col, test_sparse) if args['hypertune']: # write test_rmse metric for hyperparam tuning util.write_hptuning_metric(args, test_rmse) tf.logging.info('train RMSE = %.2f' % train_rmse) tf.logging.info('test RMSE = %.2f' % test_rmse)
def main(): # Creates the initial training sample and trains the model # test_model will then make the model play games to determine how well it scores training_data = initial_training_data() model = train_model(training_data) test_model(model)
import io import model import pandas as pd data_path = "../data/train" model_output = "../output" if __name__ == "__main__": my_model = model.train_model(data_path) model.save_model(model_output, my_model) my_model = model.load_model(model_output) with open("../../data/smoketest/iris.csv", "r") as file: test_data = file.read() data_in = io.StringIO(test_data) predictions = model.predict(data_in, my_model) out = io.StringIO() pd.DataFrame({ 'results': predictions }).to_csv(out, header=False, index=False) print(out.getvalue())
if __name__ == "__main__": assert (sys.argv[1]=="-d" or sys.argv[1]=="-t"), "-d dynet / -t theano" if sys.argv[1]=='-t': print 'using theano' train_model( max_epochs = 50, optimizer = adagrad, batch_size = 256, ndims = 50, nhiddens = 50, dropout_rate = 0.2, regularization = 0.000001, margin_loss_discount = 0.2, max_word_len = 4, load_params = None, resume_training = False, start_point = 1, max_sent_len = 60, beam_size = 4, shuffle_data = True, train_file = '../data/pku_train', dev_file = '../data/pku_dev', lr = 0.2, pre_training = '../w2v/c_vecs_50' ) else: print 'using dynet' from dy_model import dy_train_model dy_train_model( max_epochs = 50,
plist = [{'params': model_ft.layer0.parameters()}, {'params': model_ft.layer1.parameters()}, {'params': model_ft.layer2.parameters()}, {'params': model_ft.layer3.parameters(), 'lr': lr_min}, {'params': model_ft.layer4.parameters(), 'lr': lr_min, 'weight': 0.001}, {'params': model_ft.last_linear.parameters(), 'lr': lr_max} ] optimizer_ft = optim.Adam(plist, lr=0.001) model_ft, optimizer_ft = amp.initialize(model_ft, optimizer_ft, opt_level="O1", verbosity=0) lr_sch = lr_scheduler.ReduceLROnPlateau(optimizer_ft, verbose=True, factor=0.3, mode="max", patience=1, threshold=0.01) dataset_sizes = {} dataset_sizes["train"] = len(train_dataset) dataset_sizes["val"] = len(valid_dataset) data_loader = {} data_loader["train"] = train_dataset_loader data_loader["val"] = valid_dataset_loader model_ft = train_model(model_ft, data_loader, dataset_sizes, device, optimizer_ft, lr_sch, num_epochs=EPOCHS, fold_name=FOLD_NAME, use_amp=True) torch.save(model_ft.state_dict(), os.path.join(FOLD_NAME, "model.bin"))
def read_batched_ndarrays(n_train, n_valid, batch_size, max_seq_len, n_dims): """Training a model with Dataset created using a generator of batched data.""" train_model( batched_generator_dataset(n_train, batch_size, max_seq_len, n_dims), batched_generator_dataset(n_valid, batch_size, max_seq_len, n_dims))
# w2v = w2v_input('/media/wanghao/0DD813960DD81396/work/summer_try/glove.6B/glove.6B.100d.txt') w2v = w2v_input("/home/wanghao/workspace/cl/w2v.txt") # print(len(w2v)) # print(w2v['this']) wordmat = w2v_init(words, w2v) print('Read Embedding Mat: Done!') # print(w2v[words[senmat[2][4]]]) # print(wordmat[senmat[2][4]]) testq, testa, intestq, intesta = read_test('/home/wanghao/workspace/cl/dev.txt', sen_no) test2q, test2a, intest2q, intest2a = read_test('/home/wanghao/workspace/cl/test.txt', sen_no) print(len(testq)) t = train_model(max_document_length, wordmat, sen_q, sen_ar, sen_aw, senmat, lenwords, testq, testa, intestq, intesta, test2q, test2a, intest2q, intest2a) # print(sen_q[0], sentences[sen_q[0]]) # print(sen_ar[0], sentences[sen_ar[0]]) # print(sen_aw[0], sentences[sen_aw[0]])
"-sh", "--show_history", nargs=1, help="Show model's training history. -sh history_json_path") args = parser.parse_args() # model_1 - epoch 100, batch_size = 4, optimizer='adadelta', autoencoder # model_2 - epoch 100, batch_size = 4, optimizer='adadelta', unet # model_3 - epoch 200, batch_size = 4, optimizer='adadelta', autoencoder # model_4 - epoch 200, batch_size = 4, optimizer='adadelta', unet if (args.train): from model import train_model args = args.train train_model(args[0], args[1], args[2]) if (args.predict_single): from model import predict_image args = args.predict_single predict_image(args[0], args[1]) if (args.predict_batch): # WIP, not fully functional from model import predict_batch args = args.predict_batch predict_batch(args[0], args[1]) if (args.predict_video): from model import predict_video args = args.predict_video predict_video(args[0], args[1]) if (args.segment_background): from utilities import segment_background args = args.segment_background
train_aspects = example_reader.pad_aspect_index(train_aspect_text_inputs.tolist(), max_length=22) test_aspects = example_reader.pad_aspect_index(test_aspect_text_inputs.tolist(), max_length=22) model = m.build_model(max_len=78, aspect_max_len=22, embedding_matrix=embedding_matrix, position_embedding_matrix=position_matrix, num_words=5144) evaluator = Evaluator(true_labels=test_true_labels, sentences=test_sentence_inputs, aspects=test_aspect_text_inputs) epoch = 1 while epoch <= 50: model = m.train_model(sentence_inputs=train_sentence_inputs, position_inputs= train_positions, aspect_input=train_aspects, labels=train_aspect_labels, model=model) results = m.get_predict(sentence_inputs=test_sentence_inputs, position_inputs=test_positions, aspect_input=test_aspects, model=model) print("\n--epoch"+str(epoch)+"--") F, acc = evaluator.get_macro_f1(predictions=results, epoch=epoch) if epoch % 2 == 0: print("current max f1 score"+str(evaluator.max_F1)) print("max f1 is gained in epoch"+str(evaluator.max_F1_epoch)) print("current max acc"+str(evaluator.max_acc)) print("max acc is gained in epoch"+str(evaluator.max_acc_epoch)) print("happy ending")
# trainY = keras.utils.to_categorical(trainY, n_classes) # validY = keras.utils.to_categorical(validY, n_classes) trainX = train_seq[:, -8:-4, :] # [:,-8:-4,:] [:,-8:-4,1:3] trainY = train_seq[:, -4:, 1:3] validX = valid_seq[:, -8:-4, :] # [:,-8:-4,:] [:,-8:-4,1:3] validY = valid_seq[:, -4:, 1:3] print('trainX.shape:', trainX.shape) print('trainY.shape:', trainY.shape) print('validX.shape:', validX.shape) print('validY.shape:', validY.shape) print('trainX[0]\n', trainX[0]) print('trainY[0]\n:', trainY[0]) # 搭建网络训练模型 model.helloworld() network = model.build_model(n_step=n_step, n_input=n_input) history = model.train_model(trainX, trainY, validX, validY, network) model.plt_result(history) # model.model_evaluate() # model.model_predict() # truth,predict = train(train_seq,valid_seq) # rmse = evaluate(truth,predict) # print("rmse = ",rmse) # draw_route(truth,predict) # generate_result(predict,results_folder_path) elif mode == "test": #test_path = "/home/data" #测试集路径。存储形式同“训练集”,下含Track和Image两个文件夹 track_folder_path = "dataset/Test/Track" # #测试集台风路径文件路径 images_folder_path = "dataset/Test/Image" #测试集卫星云图文件路径 results_folder_path = "result/result.csv" #结果输出文件路径 model.helloworld() predict = test_predict(images_folder_path, track_folder_path)
from config import INTENTS from model import train_model, load_model, save_model from preprocessing import preprocess_messages, tokenize_messages import numpy as np save_model(train_model()) model = load_model() while True: sentence = input("Bitte gib deine Nachricht ein:") input_data = preprocess_messages(tokenize_messages([sentence])) prediction = model.predict(input_data)[0] index = np.argmax(prediction) print(prediction) print("Erkannter Intent für Satz '{}': {} ({:.2%})".format( sentence, INTENTS[index], prediction[index]))
''' If we have at least that many records, let's actually train a model. ''' feature_data.index = range(record_count) target_data.index = range(record_count) # Subset feature labels to exclude our indices if num_dockets > min_record_count and model == None: # Set the excluded feature labels feature_labels = [label for label in feature_data.columns.tolist() if label not in bad_feature_labels] # Train the model on the data model = train_model(feature_data[feature_labels], target_data[0].apply(int).tolist(), search_parameters) elif num_dockets > min_record_count and num_dockets % 100 == 0: print((docket_id, num_dockets)) # Train the model on the data model = train_model(feature_data[feature_labels], target_data[0].apply(int).tolist(), search_parameters) # Output the feature weight data feature_weight_df = pandas.DataFrame(feature_weights, columns=feature_labels) # Track the case assessment case_assessment = []
# print('data_directory = {!r}'.format(parse_results.data_directory)) data_dir = parse_results.data_directory save_dir = parse_results.save_dir arch = parse_results.arch learning_rate = float(parse_results.learning_rate) hidden_units = int(parse_results.hidden_units) epochs = int(parse_results.epochs) gpu = parse_results.gpu # Load and preprocess data image_datasets, train_loader, valid_loader, test_loader = preprocess(data_dir) # Building and training the classifier model_init = build_model(arch, hidden_units) model, optimizer, criterion = train_model(model_init, train_loader, valid_loader, learning_rate, epochs, gpu) # Save the checkpoint model.to('cpu') model.class_to_idx = image_datasets['train'].class_to_idx checkpoint = { 'model': model, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'criterion': criterion, 'epochs': epochs, 'class_to_idx': model.class_to_idx } torch.save(checkpoint, save_dir + '/checkpoint.pth') if save_dir == ".":
create_sequences, train_test, generated) if __name__ == "__main__": values_dt = pd.read_csv( '../temp_ds/Power-Networks-LCL-June2015(withAcornGps)v2_2.csv', delimiter=',') values_dt = np.asarray(values_dt['KWH/hh (per half hour) '].dropna( how='any', axis=0)) values_dt[np.where(values_dt == 'Null')] = -1 values_dt = values_dt.astype(np.float32) splited = split_data(values_dt, 50) #Нарезаем на 50 батчей avg_splited = [ moving_average(splited[i], 20) for i in range(len(splited)) ] #Усредняем scalers_data = np.asarray([ scaler(avg_splited[i]) for i in range(len(avg_splited)) ]) #Нормализуем datas = scalers_data[:, 1] # Данные (батчи) scalers = scalers_data[:, 0] # Скейлеры model = TempNN(n_features=1, n_hidden=64, seq_len=30, n_layers=1) for i, data in enumerate(datas): print("Batch №%d" % i) X_train, y_train, X_test, y_test = train_test(data) y_train = torch.reshape(y_train, (-1, 1)) y_test = torch.reshape(y_test, (-1, 1)) model, train_hist, test_hist = train_model(model, X_train, y_train, X_test, y_test) torch.save(model.state_dict(), "../models/energy_model.pth")
if args.resume: with open(args.model_path, 'rb') as f: model = torch.load(f) else: model = model.lm(n_vocab, args.d_embed, args.rnn_type, args.d_hidden, args.n_layers, args.p_dropout) if args.cuda: model.cuda() if args.train_path: if args.valid_path: valid = data.loader(args.valid_path, vocab=train.vocabulary) else: valid = None model.train_model(args.model_path, train, valid, args.epochs, args.batch_size, args.bptt, args.learning_rate) with open(args.model_path, 'rb') as f: model = torch.load(f) if args.test_path: test = data.loader(args.test_path, vocab=train.vocabulary) criterion = nn.CrossEntropyLoss() test_loss = model.evaluate(criterion, test, args.bptt, args.batch_size) print('=' * 89) print( '| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format( test_loss, math.exp(test_loss))) print('=' * 89) if args.input_path: with open(args.input_path, 'r') as f:
def read_unbatched_tfrecord(n_train, n_valid, batch_size, max_seq_len, n_dims): """Training a model with Dataset created using a TFRecord file.""" map(os.remove, glob.glob('*.tfr')) # Remove any existing TFRecord files train_model(tfrecord_dataset(n_train, batch_size, max_seq_len, n_dims), tfrecord_dataset(n_valid, batch_size, max_seq_len, n_dims))
def main(cfg): # 病理画像のデータのリストを取得 data_dir = make_data_path_list() # データセットの作成 dataset = PathologicalImage(file_list=data_dir, transform=ImageTransform(cfg.size), num_pixels=cfg.num_pixels) # 学習用データの枚数を取得 train_size = int(len(dataset) * cfg.rate) # 検証用のデータの枚数を取得 val_size = len(dataset) - train_size # データセットの分割 train_dataset, val_dataset = data.random_split(dataset, [train_size, val_size]) # 動作確認 print("入力画像サイズ:" + str(train_dataset.__getitem__(0)[0].size())) print("教師データサイズ:" + str(train_dataset.__getitem__(0)[1].shape)) # 学習用のDataLoaderを作成 train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True) # 検証用のDataLoaderを作成 val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=cfg.batch_size, shuffle=False) # 動作確認 batch_iterator = iter(train_dataloader) inputs, labels = next(batch_iterator) log.info("-----Image and label shape of dataloader-----") log.info("入力データ:" + str(inputs.size())) log.info("入力ラベル:" + str(labels.shape)) # GPU初期設定 # ネットワークモデル(自作FCNs)をimport device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # ネットワークをGPUへ net = FCNs() net.to(device) torch.backends.cudnn.benchmark = True log.info("-----Constitution of network-----") log.info(net) # 損失関数の設定 criterion = nn.MSELoss() # 最適化手法の設定 optimizer = optim.SGD(net.parameters(), lr=cfg.SGD.lr, momentum=cfg.SGD.momentum) log.info("-----Details of optimizer function-----") log.info(optimizer) # 損失値を保持するリスト train_loss = [] val_loss = [] # 学習 for epoch in range(cfg.num_epochs): log.info("Epoch {} / {} ".format(epoch + 1, cfg.num_epochs)) log.info("----------") # 学習 train_history = train_model(net, train_dataloader, criterion, optimizer) # 学習したlossのリストを作成 train_loss.append(train_history) # 検証 val_history = val_model(net, val_dataloader, criterion) # 検証したlossのリスト作成 val_loss.append(val_history) # テストと出力値保存 test_history = test_model(net, val_dataloader, criterion) # figインスタンスとaxインスタンスを作成 fig_loss, ax_loss = plt.subplots(figsize=(10, 10)) ax_loss.plot(range(1, cfg.num_epochs + 1, 1), train_loss, label="train_loss") ax_loss.plot(range(1, cfg.num_epochs + 1, 1), val_loss, label="val_loss") ax_loss.set_xlabel("epoch") ax_loss.legend() fig_loss.savefig("loss.png") # パラメータの保存 save_path = './pathological.pth' torch.save(net.state_dict(), save_path)
def run_with_predictor(): main(predictor=train_model())
sys.path.append(os.path.join(PATH,"..","..","python","tools")) import datatool net_num = '0' run = 0 model = create_model() #Use sequences 0-9 sequences = [] for i in range(10): sequences.append(i) x_tr, y_tr, x_te, y_te = datatool.get_training_data(sequences, training_ratio=(1)) score, history = train_model(model, x_tr, y_tr, x_te, y_te, os.path.join(PATH, "train_"+str(run)+".h5")) with open(os.path.join(PATH, "history_"+str(run)+".json"), 'w') as f: json.dump(score, f, indent=4) json.dump(history, f, indent=4) plt.plot(history.history['translation_mean_absoulte_error']) plt.plot(history.history['val_translation_mean_absoulte_error']) plt.title('model mean_absoulte_error per epoch') plt.ylabel('translation mean absoulute error') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() plt.plot(history.history['rotation_mean_absoulte_error']) plt.plot(history.history['val_rotation_mean_absoulte_error'])
maxlen = 25 # maximum length for each sentence. max_features = 25000 # length of the vocabulary. batch_size = 32 nb_epoch = 4 additional_num_words = 2 # "UNK" and "PADDING" (X_train, y_train), (_, _), word_idx = data.read(fn, 0.0, maxlen, max_features, problem_type) print >> sys.stderr, 'X_train shape:', X_train.shape max_features = min(max_features, len(word_idx) + additional_num_words) if problem_type == 'regression': model = create_regression_model(maxlen, max_features, word_idx, use_pretrained_embeddings) else: model = create_logistic_model(maxlen, max_features, word_idx, use_pretrained_embeddings) train_model(model, X_train, y_train, batch_size, nb_epoch) model.save_weights(os.path.join(model_output_dir, 'weights.h5'), overwrite=True) print >> sys.stderr, "Weights are written" cPickle.dump(word_idx, codecs.open(os.path.join(model_output_dir, 'word_idx.json'), 'w', encoding='utf8')) print >> sys.stderr, "word_idx dict is written" architecture = model.to_json() open(os.path.join(model_output_dir, 'architecture.json'), 'w').write(architecture) print >> sys.stderr, "Architecture is written"
import csv import os import pickle import pandas as pd from flask import * from flask import Flask, jsonify, request from flask_restful import Api, Resource from model import train_model app = Flask(__name__) api = Api(app) if not os.path.isfile('model.pkl'): train_model() model = pickle.load(open('model.pkl', 'rb')) def feature_engineering(df): df['dow'] = df['click_time'].dt.dayofweek.astype('uint16') df['doy'] = df['click_time'].dt.dayofyear.astype('uint16') df['hour'] = df['click_time'].dt.hour.astype('uint16') features_clicks = ['ip', 'app', 'os', 'device'] for col in features_clicks: col_count_dict = dict(df[[col]].groupby(col).size().sort_index()) df['{}_clicks'.format(col)] = df[col].map(col_count_dict).astype( 'uint16') features_comb_list = [('app', 'device'), ('ip', 'app'), ('app', 'os')]
url = '/home/medialab/Zhewei/MICCAI_Les_2017_Process/Unet_3D_Les/BAK/' trainlinks, trainannotlinks, testlinks, testannotlinks = readfile(url+'link/UnetSample_train.txt',\ url+'link/UnetLabel_train.txt', \ url+'link/UnetSample_test.txt', \ url+'link/UnetLabel_test.txt') conf = config trainPair = data_generator(trainlinks, trainannotlinks, conf, conf["batch_size"]) validPair = data_generator(testlinks, testannotlinks, conf, conf["batch_size"], rdm=False) train_generator = trainPair validation_generator = validPair nb_train_samples = 640 nb_test_samples = 260 """ model = unet_model_3d(input_shape=config["input_shape"], downsize_filters_factor=config["downsize_nb_filters_factor"], pool_size=config["pool_size"], n_labels=config["GT_class"], initial_learning_rate=config["initial_learning_rate"]) # run training # model = load_old_model(config["model_file"]) train_model(model=model, model_file=config["model_file"], training_generator=train_generator, validation_generator=validation_generator, steps_per_epoch=nb_train_samples, validation_steps=nb_test_samples, initial_learning_rate=config["initial_learning_rate"], learning_rate_drop=config["learning_rate_drop"], learning_rate_epochs=config["decay_learning_rate_every_x_epochs"], n_epochs=config["n_epochs"]) """
exp_directory = './Model' exp_directory = Path(exp_directory) if not exp_directory.exists(): exp_directory.mkdir() model = m.createDeepLabv3() # Specify the loss function criterion = torch.nn.MSELoss(reduction='mean') # Specify the optimizer with a lower learning rate optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) # Specify the evaluation metrics metrics = {'f1_score': f1_score, 'auroc': roc_auc_score} # Create the dataloader dataloaders = get_dataloader_single_folder( './Data', 'Image', [ 'Eyes_dorsal', 'Outline_dorsal', 'Outline_lateral', 'Ov_lateral', 'Heart_lateral', 'Yolk_lateral' ], batch_size=2) _ = m.train_model(model, criterion, dataloaders, optimizer, bpath=exp_directory, metrics=metrics, num_epochs=1)
def train_model_main(input_file, iteration, checkpoint, verbose): # ---------------- Start ------------------------- misc.print_and_flush(f"train_model {iteration}: Start") start = time.process_time() inputs = misc.read_json(input_file) path_data = inputs["path_data"] path_model = inputs["path_model"] batch_size = inputs["batch_size"] model_args = misc.read_json(inputs["model_parameters"]) board_params = inputs["board_params"] path_board = board_params["path_board"] epochs = inputs["epochs"] eval_every = inputs["eval_every"] # ---------------- Load model ------------------------- move_types = ['initialPick', 'initialFortify', 'startTurn', 'attack', 'fortify'] # Create Board world = World(path_board) # Set players pR1, pR2 = agent.RandomAgent('Red'), agent.RandomAgent('Blue') players = [pR1, pR2] # Set board # TODO: Send to inputs prefs = board_params board_orig = Board(world, players) board_orig.setPreferences(prefs) num_nodes = board_orig.world.map_graph.number_of_nodes() num_edges = board_orig.world.map_graph.number_of_edges() if verbose: misc.print_and_flush("Creating model") net = GCN_risk(num_nodes, num_edges, model_args['board_input_dim'], model_args['global_input_dim'], model_args['hidden_global_dim'], model_args['num_global_layers'], model_args['hidden_conv_dim'], model_args['num_conv_layers'], model_args['hidden_pick_dim'], model_args['num_pick_layers'], model_args['out_pick_dim'], model_args['hidden_place_dim'], model_args['num_place_layers'], model_args['out_place_dim'], model_args['hidden_attack_dim'], model_args['num_attack_layers'], model_args['out_attack_dim'], model_args['hidden_fortify_dim'], model_args['num_fortify_layers'], model_args['out_fortify_dim'], model_args['hidden_value_dim'], model_args['num_value_layers'], model_args['dropout']) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') net.to(device) optimizer = torch.optim.Adam(net.parameters(), lr=0.001) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5) criterion = TPT_Loss #state_dict = model.load_dict(os.path.join(path_model, checkpoint), device = 'cpu', encoding = 'latin1') #net.load_state_dict(state_dict['model']) #optimizer.load_state_dict(state_dict['optimizer']) #scheduler.load_state_dict(state_dict['scheduler']) load_path = os.path.join(path_model, checkpoint) if checkpoint else None # This is used only at the beginning. Then the model that is loaded is trained and saved at each time. # We avoid reloading the last saved model # Train network on dataset if verbose: misc.print_and_flush("Training network") shuffle(move_types) for j, move_type in enumerate(move_types): if verbose: misc.print_and_flush(f"\tTraining {j}: {move_type}") save_path = f"{path_model}/model_{iteration}_{j}_{move_type}.tar" root_path = f'{path_data}/{move_type}' if len(os.listdir(os.path.join(root_path, 'raw')))<batch_size: continue risk_dataset = RiskDataset(root = root_path) # TODO: add validation data loader = G_DataLoader(risk_dataset, batch_size=batch_size, shuffle = True) if verbose: misc.print_and_flush(f"\tTrain on {root_path}, model = {save_path}") train_model(net, optimizer, scheduler, criterion, device, epochs = epochs, train_loader = loader, val_loader = None, eval_every = eval_every, load_path = load_path, save_path = save_path) load_path = None # The model is already in memory misc.print_and_flush(f"train_model: Total time taken -> {round(time.process_time() - start,2)}")
root_path = f'{path_data}/{move_type}' if len(os.listdir(os.path.join(root_path, 'raw'))) < batch_size: continue risk_dataset = RiskDataset(root=root_path) # TODO: add validation data loader = G_DataLoader(risk_dataset, batch_size=batch_size, shuffle=True) if verbose: print(f"\tTrain on {root_path}, model = {save_path}") train_model(net, optimizer, scheduler, criterion, device, epochs=10, train_loader=loader, val_loader=None, eval_every=3, load_path=None, save_path=save_path) print(f"Time taken: {round(time.perf_counter() - start,2)}") print("Building expert") # build expert with trained net apprentice = agent.NetApprentice(net) expert = build_expert_mcts(apprentice) expert.num_MCTS_sims = expert_mcts_sims