def setUp(self): self.train_set, _ = dta.get_client_data( 'emnist', 'example', { 'supervised': 0.0, 'unsupervised': 0.0 }, ) self.train_set = dta.get_sample_client_data(self.train_set, 2, 8) classifier_ph = autoencoder_ph = { 'dataset': 'emnist', 'optimizer': 'SGD', 'learning_rate': 10.0 } self.classifier = mdl.DenseSupervisedModel(classifier_ph) self.autoencoder = mdl.DenseAutoencoderModel(autoencoder_ph) self.dataloader_classifier = dta.DataLoader( self.classifier.preprocess_emnist, num_epochs=1, shuffle_buffer=500, batch_size=2, learning_env='federated') self.dataloader_autoencoder = dta.DataLoader( self.autoencoder.preprocess_emnist, num_epochs=1, shuffle_buffer=500, batch_size=2, learning_env='federated')
def create_dataloaders(LABEL_MODE, TRAIN_FOLDS, TRAIN_SCENES, BATCHSIZE, TIMESTEPS, EPOCHS, NFEATURES, NCLASSES, VAL_FOLDS, VAL_STATEFUL): train_loader = dataloader.DataLoader('train', LABEL_MODE, TRAIN_FOLDS, TRAIN_SCENES, batchsize=BATCHSIZE, timesteps=TIMESTEPS, epochs=EPOCHS, features=NFEATURES, classes=NCLASSES) train_loader_len = train_loader.len() print('Number of batches per epoch (training): ' + str(train_loader_len)) val_loader = dataloader.DataLoader('val', LABEL_MODE, VAL_FOLDS, TRAIN_SCENES, epochs=EPOCHS, batchsize=BATCHSIZE, timesteps=TIMESTEPS, features=NFEATURES, classes=NCLASSES, val_stateful=VAL_STATEFUL) val_loader_len = val_loader.len() print('Number of batches per epoch (validation): ' + str(val_loader_len)) return train_loader, val_loader
def train(config): cuda = torch.cuda.is_available() device = torch.device("cuda" if cuda else "cpu") dehaze_net = net.AODNet().to(device) dehaze_net.apply(weights_init) train_dataset = dataloader.DataLoader(config.path_clearimg, config.path_hazyimg) val_dataset = dataloader.DataLoader(config.path_clearimg, config.path_hazyimg, mode="val") train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config.train_batch_size, shuffle=True, num_workers=config.num_workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=config.val_batch_size, shuffle=True, num_workers=config.num_workers, pin_memory=True) # criterion = nn.MSELoss().to(device) criterion = pytorch_ssim.SSIM(window_size=11) optimizer = torch.optim.Adam(dehaze_net.parameters(), lr=config.lr, weight_decay=config.weight_decay) # optimizer = torch.optim.SGD(dehaze_net.parameters( # ), lr=config.lr, weight_decay=config.weight_decay) dehaze_net.train() for epoch in range(config.num_epochs): for iteration, (clear_img, hazy_img) in enumerate(train_loader): clear_img = clear_img.to(device) hazy_img = hazy_img.to(device) clean_image = dehaze_net(hazy_img) loss = criterion(clean_image, clear_img) optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(dehaze_net.parameters(), config.grad_clip_norm) optimizer.step() if ((iteration + 1) % config.display_iter) == 0: print("Epoch", epoch + 1, ": Loss at iteration", iteration + 1, ":", loss.item()) # Validation Stage for iter_val, (clear_img, hazy_img) in enumerate(val_loader): clear_img = clear_img.to(device) hazy_img = hazy_img.to(device) clean_image = dehaze_net(hazy_img) torchvision.utils.save_image( torch.cat((hazy_img, clean_image, clear_img), 0), config.sample_output_folder + str(iter_val + 1) + ".jpg") torch.save(dehaze_net.state_dict(), config.snapshots_folder + "Epoch" + str(epoch + 1) + '.pt')
def renew_everything(self): # renew dataloader. self.identity_loader = DL.DataLoader(config, DL.IdentityDataset(config)) self.identity_loader.renew(min(floor(self.resl), self.max_resl)) self.loader = DL.DataLoader(config, DL.TrainingDataset(config)) self.loader.renew(min(floor(self.resl), self.max_resl)) # define tensors self.z = torch.FloatTensor(self.loader.batch_size, self.nz) self.x = torch.FloatTensor(self.loader.batch_size, 3, self.loader.im_size, self.loader.im_size) self.x_tilde = torch.FloatTensor(self.loader.batch_size, 3, self.loader.im_size, self.loader.im_size) self.real_label = torch.FloatTensor(self.loader.batch_size).fill_(1) self.fake_label = torch.FloatTensor(self.loader.batch_size).fill_(0) # enable cuda if self.use_cuda: self.z = self.z.cuda() self.x = self.x.cuda() self.x_tilde = self.x.cuda() self.real_label = self.real_label.cuda() self.fake_label = self.fake_label.cuda() torch.cuda.manual_seed(config.random_seed) # wrapping autograd Variable. self.x = Variable(self.x) self.x_tilde = Variable(self.x_tilde) self.z = Variable(self.z) self.real_label = Variable(self.real_label) self.fake_label = Variable(self.fake_label) # ship new model to cuda. if self.use_cuda: self.G = self.G.cuda() self.D = self.D.cuda() # optimizer betas = (self.config.beta1, self.config.beta2) if self.optimizer == 'adam': self.opt_g = Adam(filter(lambda p: p.requires_grad, self.G.parameters()), lr=self.lr, betas=betas, weight_decay=0.0) self.opt_d = Adam(filter(lambda p: p.requires_grad, self.D.parameters()), lr=self.lr, betas=betas, weight_decay=0.0)
def __init__(self, ph): Algorithm.__init__(self, ph) self.dataloader = dta.DataLoader(self.preprocess_fn, self.ph['num_epochs'], self.ph['shuffle_buffer'], self.ph['batch_size']) self.num_epochs = self.ph['num_epochs']
def create_random(self, ): # load data for training v = loader.DataLoader() start_training = v.load_data(self.data_set, self.class_names) # shuffling the array random.shuffle(start_training) (xtrain, ytrain) = ([], []) # creating arrays for both the features and the labels for d, c in start_training: xtrain.append(d) ytrain.append(c) (xtrain, ytrain) = [np.array(lis) for lis in [xtrain, ytrain]] xtrain = np.asarray(xtrain) ytrain = np.asarray(ytrain) # 80% training and 20% testing x_train, x_test, y_train, y_test = train_test_split( xtrain, ytrain, test_size=self.num_split, random_state=0) self.sets_gets.xTest = x_test self.sets_gets.yTest = y_test xt = self.sets_gets.xTest yt = self.sets_gets.yTest # saving the testing data to pickle pickle.dump(xt, open(self.img, 'wb')) print("Random Forest X Test Saved") pickle.dump(yt, open(self.lbl, 'wb')) print("Random Forest Y Test Saved") # training the random forest model forest_classifier = RandomForestClassifier(n_estimators=2) model = forest_classifier.fit(x_train, y_train) # accuracy and loss for the model a = forest_classifier.score(x_train, y_train) print("the value of the training accuracy : " + str(a)) l = forest_classifier.score(x_test, y_test) print("the value of the testing accuracy : " + str(l)) # confusion matrics y_pred = forest_classifier.predict(x_test) print("Classification Report:") print(metrics.classification_report(y_test, y_pred)) print("Confusion Matrix:") print(metrics.confusion_matrix(y_test, y_pred)) # saving the random forest model to pickle pickle.dump(model, open(self.model_name, 'wb')) print("Random Forest Model Saved")
def __init__(self, ph): Algorithm.__init__(self, ph) self.num_rounds = self.ph['num_rounds'] self.num_clients_per_round = self.ph['num_clients_per_round'] self.dataloader = dta.DataLoader(self.preprocess_fn, self.ph['num_epochs'], self.ph['shuffle_buffer'], self.ph['batch_size'], 'federated')
def initialize(self, opt): BaseDataLoader.initialize(self, opt) root_path = "traindata/" self.mnistdataset = dataloader.get_training_set(root_path) self.mnistdataloader = dataloader.DataLoader( dataset=self.mnistdataset, num_workers=int(opt.nThreads), batch_size=opt.batchSize / 2, shuffle=True)
def setUp(self): ph = {'optimizer': 'SGD', 'learning_rate': 10.0, 'dataset': 'cifar100'} keras_model_fn = mdl.SimpleRotationSelfSupervisedModel(ph) preprocess_fn = getattr(keras_model_fn, 'preprocess_{}'.format(ph['dataset'])) dataloader = dta.DataLoader(preprocess_fn, num_epochs=1, shuffle_buffer=1, batch_size=20, learning_env='federated') train_client_data, _ = dta.get_client_data(ph['dataset'], 'example', { 'supervised': 0.0, 'unsupervised': 0.0 }, sample_client_data=False) sample_batch = dataloader.get_sample_batch(train_client_data) model_fn = functools.partial(keras_model_fn.create_tff_model_fn, sample_batch) iterative_process = tff.learning.build_federated_averaging_process( model_fn) state = iterative_process.initialize() sample_clients = train_client_data.client_ids[:5] federated_train_data = dataloader.make_federated_data( train_client_data, sample_clients) state, _ = iterative_process.next(state, federated_train_data) self.old_model = keras_model_fn() self.old_model.build(sample_batch) tff.learning.assign_weights_to_keras_model(self.old_model, state.model) self.tmp_dir = 'tests/tmp/' if not os.path.isdir(self.tmp_dir): os.mkdir(self.tmp_dir) self.model_fp = os.path.join(self.tmp_dir, 'model.h5') keras_model_fn.save_model_weights(self.model_fp, state, sample_batch) self.new_model = keras_model_fn.load_model_weights(self.model_fp) ph = { 'optimizer': 'SGD', 'learning_rate': 10.0, 'dataset': 'cifar100', 'pretrained_model_fp': self.model_fp } self.transfer_model = mdl.SimpleRotationSupervisedModel(ph)()
def setUp(self): self.train_set, _ = dta.get_client_data('emnist', 'example', {'supervised':0.0, 'unsupervised':0.0} ) self.train_set = dta.get_sample_client_data(self.train_set, 2, 8) self.dataloader_classifier = dta.DataLoader(dta.preprocess_classifier, num_epochs = 1, shuffle_buffer = 500, batch_size = 2 ) self.dataloader_autoencoder = dta.DataLoader(dta.preprocess_autoencoder, num_epochs = 1, shuffle_buffer = 500, batch_size = 2 )
def get_price(self, symbols, dates): """Read stock data (price) for given symbols from files.""" df = pd.DataFrame(index=dates) for symbol in symbols: df_temp = dataloader.DataLoader().loaddata( self.__symbol_to_path(symbol)) df_temp = pd.DataFrame(df_temp, columns={'price'}) df_temp = df_temp.rename(columns={'price': symbol}) df = df.join(df_temp) df = df.dropna() return df
def test(model_dict, using_cuda=True): if using_cuda: net = Net().cuda() else: net = Net() net.load_state_dict(torch.load(model_dict)) dataset = dataloader.DataLoader("test_set.pkl", batch_size=1, using_cuda=using_cuda) count = 0 for i, batch in enumerate(dataset): X = batch["feature"] y = batch["class"] y_pred, _ = net(X) p, idx = torch.max(y_pred.data, dim=1) count += torch.sum(torch.eq(idx.cpu(), y.data.cpu())) print("accuracy: %f" % (count / dataset.num))
def main(): mscoco = dataloader.DataLoader(data_dir = './MSCOCO-IC/') import sys kind = sys.argv[1] # train NUM_CLASSES = 10 resnet = models.resnet18(pretrained=True) num_ftrs = resnet.fc.in_features resnet.fc = torch.nn.Linear(num_ftrs, NUM_CLASSES) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(resnet.parameters(), lr=0.001, momentum=0.9) exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) #resnet.load_state_dict(torch.load('./coco_'+kind+'.pth')) train_model(resnet, mscoco, criterion, optimizer, 20, './coco_'+kind+'.pth', kind)
def run(): df = pd.read_csv(CONFIG.INPUT_PATH, encoding="latin-1") df.loc[:, "Sentence #"] = df["Sentence #"].fillna(method="ffill") pos_lb = LabelEncoder() pos_lb = pos_lb.fit(df['POS'].values) df['POS'] = pos_lb.transform(df['POS'].values) sentence = df.groupby('Sentence #')['Word'].apply(list).values pos = df.groupby('Sentence #')["POS"].apply(list).values x_val = sentence[:int(0.1 * len(sentence))] y_val = pos[:int(0.1 * len(sentence))] print('--------- [INFO] TOKENIZING --------') train_loader = dataloader.DataLoader(sentence, pos, CONFIG.Batch_size) pickle.dump(pos_lb, open('input/pos_lb.pickle', 'wb')) pickle.dump(train_loader.vocab.word_to_idx, open('input/word_to_idx.pickle', 'wb')) x_val = train_loader.vocab.numericalize(x_val) x_val = keras.preprocessing.sequence.pad_sequences( x_val, padding='post', value=train_loader.vocab.word_to_idx["<PAD>"]) y_val = keras.preprocessing.sequence.pad_sequences(y_val, padding='post') vocab_size = len(train_loader.vocab.word_to_idx) classes = len(list(pos_lb.classes_)) model = NERModel.NERModel(vocab_size=vocab_size, num_classes=classes) model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['sparse_categorical_accuracy']) print(f'------- [INFO] STARTING TRAINING -------') model.fit(train_loader, epochs=CONFIG.Epochs, batch_size=CONFIG.Batch_size, validation_data=(x_val, y_val)) model.save(CONFIG.MODEL_PATH)
def load_dataset(tensors): transform = transforms.Compose([ transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) dataset = datasets.ImageFolder(root=FLAGS.data_dir, transform=transform) tensor_type = 'float16' loader = dataloader.DataLoader( dataset, batch_size=FLAGS.micro_batch_size * FLAGS.batches_per_step, tensor_type=tensor_type, shuffle=True, num_workers=FLAGS.num_workers, drop_last=True ) # In the case there is not sufficient data in last batch drop it return DataSet(tensors, FLAGS.micro_batch_size, FLAGS.batches_per_step, loader, np.float16)
def inference(): device = "cuda:0" if torch.cuda.is_available() else "cpu" save_file = input("save model name : ") try: if torch.cuda.is_available(): model = torch.load(save_file, map_location={"cpu": "cuda:0"}) else: model = torch.load(save_file, map_location={"cuda:0": "cpu"}) print("Success loading model") except IOError: print("Couldn't find model") sys.exit(0) print("best epoch was {}".format(model.info_dict['epoch'])) # 1783 : length of test data set test_data_loader = dataloader.DataLoader(1783, test=True) model.eval() with torch.no_grad(): X, _ = test_data_loader.get_batch() X = X.to(device) output = model(X) utils.generate_csv(output)
import ABot_Decoder import QBot import QBot_Encoder import QBot_Decoder import discriminator as Discriminator torch.backends.cudnn.enabled = False random.seed(32) np.random.seed(32) torch.manual_seed(7) torch.cuda.manual_seed_all(7) #Load Data dialog_loc = '../../../chat_processed_data.h5' param_loc = '../../../chat_processed_params.json' image_loc = '../../../data_img.h5' data = dataloader.DataLoader(dialog_loc, image_loc, param_loc) print("Done: Data Preparation") #CUDA USE_CUDA = True gpu = 0 #Parameters params = {} params['batch_first'] = False params['num_layers'] = 2 params['hidden_dim'] = 512 params['embed_size'] = 300 params['vocab_size'] = len(data.ind2word.keys()) params['embedding_size'] = 300 params['vgg_out'] = 4096
download=not args.no_download_dataset) sensor_size = dataset.sensor_size elif args.dataset == "dvs_gesture": transform = tonic.transforms.Compose([ tonic.transforms.Downsample(spatial_factor=0.25)]) dataset = tonic.datasets.DVSGesture(save_to='./data', train=True, transform=transform, download=not args.no_download_dataset) sensor_size = (32, 32) polarity = True else: raise RuntimeError("Unknown dataset '%s'" % args.dataset) # Create loader start_processing_time = perf_counter() data_loader = dataloader.DataLoader(dataset, shuffle=True, batch_size=batch_size, sensor_size=sensor_size, polarity=polarity, dataset_slice=dataset_slice) end_process_time = perf_counter() print("Data processing time:%f ms" % ((end_process_time - start_processing_time) * 1000.0)) # Calculate number of input neurons from sensor size num_input_neurons = np.product(sensor_size) if polarity: num_input_neurons *= 2 # Calculate number of valid outputs from classes num_outputs = len(dataset.classes) # Round up to power-of-two num_output_neurons = int(2**(np.ceil(np.log2(num_outputs))))
from mxnet import nd, autograd, gluon import dataloader from sklearn.model_selection import train_test_split from matplotlib import pyplot as plt from datetime import datetime #Set Contexts ctx = mx.gpu() if mx.test_utils.list_gpus() else mx.cpu() data_ctx = ctx model_ctx = ctx #load the data num_inputs = 784 batch_size = 64 num_instances = 60000 data = dataloader.DataLoader() train_data,train_labels = data.load_data() test_data,test_labels = data.load_data(mode = 'test') X_train, X_val, y_train, y_val = train_test_split(train_data, train_labels, test_size=0.30, random_state=42) train_data = [] for index,data in enumerate(X_train): temp = y_train[index] train_data.append((data,temp)) num_instances = len(train_data) val_data = [] for index,data in enumerate(X_val): temp = y_val[index] val_data.append((data,temp))
name = 'data/seq/test.pickle' iname = 'data/seq/test_in.pickle' oname = 'data/seq/test_out.pickle' m = Mode.test elif mode == 'valid': name = 'data/seq/valid.pickle' iname = 'data/seq/valid_in.pickle' oname = 'data/seq/valid_out.pickle' m = Mode.train else: exit(1) # load data train = DL.DataLoader(m) # test = DL.DataLoader(Mode.test) # for validation data if mode == 'train': for k in train.keys(): train[k] = train[k][76:] elif mode == 'valid': for k in train.keys(): train[k] = train[k][:76] # get time data output_train = train.pop('output') # output_test = test.pop('output')
if "dev.conllu" in file: dev_path = treebank + "/" + file if "test.conllu" in file: test_path = treebank + "/" + file if train_path is None: continue language_fullname = "_".join(os.path.basename(treebank).split("_")[1:]) lang_full = lang f = train_path.strip() i = 0 data = pyconll.load_from_file(f"{f}") data_loader = dataloader.DataLoader(args, relation_map) inputFiles = [train_path, dev_path, test_path] data_loader.readData(inputFiles) train_features, train_output_labels = data_loader.getBinaryFeatures( train_path, type="train", p=args.percent, shuffle=True) if dev_path: dev_features, dev_output_labels = data_loader.getBinaryFeatures( dev_path, type="dev", p=1.0, shuffle=False) test_features, test_output_labels = data_loader.getBinaryFeatures( test_path, type="test", p=1.0, shuffle=False) for feature in args.features: if feature in test_features and feature in train_features: try:
iname = '{}/{}/in.pickle'.format(prefix, mode) oname = '{}/{}/out.pickle'.format(prefix, mode) mname = '{}/{}/mask.pickle'.format(prefix, mode) sname = '{}/{}/swa.pickle'.format(prefix, mode) if mode == 'train': m = Mode.train elif mode == 'test': m = Mode.test elif mode == 'valid': m = Mode.train else: exit(1) # load data data = DL.DataLoader(m) print(data.keys()) # for validation data if mode == 'train': for k in data.keys(): data[k] = data[k][76:] elif mode == 'valid': for k in data.keys(): data[k] = data[k][:76] # get time data output = data.pop('output') input = data.pop('input') swa = data.pop('SWA') print(np.array(swa).shape)
# Parse command line options (options, args) = parser.parse_args() # Assert if the dataset is within the predefined possible options datasetDict = {"MNIST": 1, "CIFAR-10": 2} if options.dataset not in datasetDict: print ("Error: Dataset not within the predefined possible options (%s)" % str(datasetDict.keys())) exit (-1) else: # Add the dataset name at the end of the model options.logDir = options.logDir + "-" + options.dataset options.checkpointDir = options.checkpointDir + "-" + options.dataset # Import dataset dataloader = dataloader.DataLoader(options.dataset, one_hot=False) options.numTrainingInstances = dataloader.train.num_examples options.numValidationInstances = dataloader.validation.num_examples options.numTestInstances = dataloader.test.num_examples print ("Training Instances: %d | Validation Instances: %d | Test Instances: %d" % (options.numTrainingInstances, options.numValidationInstances, options.numTestInstances)) options.imageWidth = dataloader.image_width options.imageHeight = dataloader.image_height options.imageChannels = dataloader.image_channels if options.dataset == "CIFAR-10": options.numPrimaryCapsules = 64 options.numDecoderFirstHiddenLayerUnits = 2048 options.numDecoderSecondHiddenLayerUnits = 4096
def main(): args = parser.parse_args() #Create Train and Validation Loaders data_train = dataloader.DataLoader(args.dataset, args.num_classes, args, train=True) data_val = dataloader.DataLoader(args.dataset, args.num_classes, args, train=False) #Check actualy number of classes if args.num_classes == -1: args.num_classes = data_train.num_classes data_train = torch.utils.data.DataLoader(data_train, batch_size=args.batch_size, shuffle=True, num_workers=0) data_val = torch.utils.data.DataLoader(data_val, batch_size=args.batch_size, shuffle=False, num_workers=0) # Create Model model = network.Network(args.num_classes, args.projection_size, args.base_model, args.pretrained, args.normalize, args.img_size, args.layer_size).to(device) if not args.contrastive: model.convert() if args.load_model: load_model(model, args.start_epoch, args.contrastive) #Losses contrastive_loss = ContLoss(args.batch_size, args.temperature, args.sim_type) cross_entropy_loss = torch.nn.CrossEntropyLoss() #Optimizer opt, lr_scheduler = start_opt(args, model) if args.train: for process in range(2): for e in range(args.epoch): losses = [] for step, (images, labels) in enumerate(data_train): opt.zero_grad() #To adjust for dataloader outputting two images if len(images.shape) == 5: images = images.permute(1, 0, 2, 3, 4) images = torch.cat([images[0], images[1]]) #Loss bug so skip and get in next shuffled batch if images.shape[ 0] != args.batch_size * 2 and args.contrastive and process == 0: continue z = model(images) if args.contrastive and process == 0: loss = contrastive_loss(z) else: loss = cross_entropy_loss(z, labels.squeeze()) loss.backward() opt.step() losses.append(loss.item()) lr_scheduler.step() #Validate after epoch either all the time for normal model or during find layer tuning for contrastive if process == 2 and args.contrastive or not args.contrastive: validate(data_val, model, e) print("Epoch: ", e, ", Loss: ", np.mean(losses), '\n') save_model(model, e, args.contrastive) #If just resnet second stage not needed if not args.contrastive: break elif process == 0: model.convert() args.epoch = args.final_layer_epoch opt, lr_scheduler = start_opt(args, model) args.contrastive = False else: if not args.load_model: print("Define load model for test") else: validate(data_val, model, e)
def training(): train_data_loader = dataloader.DataLoader(BATCH_SIZE, test=False) eval_X, eval_y = train_data_loader.get_eval_data() eval_data_loader = dataloader.DataLoader(BATCH_SIZE, test=False, X=eval_X, y=eval_y) writer = SummaryWriter(SUMMARY_WRITER_PATH) if SCHEDULER: scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=STEP_SIZE, gamma=GAMMA) if loading: best_loss = float(model.info_dict['best_loss']) init_epoch = int(model.info_dict['epoch']) + 1 print("Best loss is {:4f}".format(best_loss)) else: best_loss = math.inf init_epoch = 1 for epoch in range(init_epoch, EPOCH_SIZE + 1): print("\n>>> Epoch {} / {}".format(epoch, EPOCH_SIZE)) # print(">>> Start training") model.train() # training mode if SCHEDULER: scheduler.step() train_losses = [] iteration = 0 while train_data_loader.next_is_available(): iteration += 1 X, y = train_data_loader.get_batch() X, y = X.to(DEVICE), y.to(DEVICE) optimizer.zero_grad() out = model(X) train_loss = nn.MSELoss()(out, y) train_losses.append(train_loss.item()) train_loss.backward() optimizer.step() if iteration == 1 and epoch % 10 == 0: utils.save_figures( X, out, "training_images/train_{}.png".format(epoch)) # print("training loss : {:12.4f}".format(train_loss), end='\r') avg_train_loss = mean(train_losses) # print("\n >>> Average training loss: {}".format(avg_train_loss)) writer.add_scalar('avg_train_loss', avg_train_loss, epoch) train_data_loader.restart(shuffle=SHUFFLE) # print(">>> Start Test") model.eval() # evaluation mode test_losses = [] val_iteration = 0 with torch.no_grad(): while eval_data_loader.next_is_available(): val_iteration += 1 X, y = eval_data_loader.get_batch() X, y = X.to(DEVICE), y.to(DEVICE) out = model(X) test_loss = nn.MSELoss()(out, y) test_losses.append(test_loss.item()) if val_iteration == 1 and epoch % 10 == 0: utils.save_figures(X, out, "test_images/test_{}.png".format(epoch)) avg_test_loss = mean(test_losses) print(">>> Average test loss: {}".format(avg_test_loss)) writer.add_scalar('avg_test_loss', avg_test_loss, epoch) eval_data_loader.restart() if avg_test_loss < best_loss: print(">>> Saving models...") best_loss = avg_test_loss save_dict = { "epoch": epoch, "best_loss": best_loss, "optimizer": optimizer.state_dict() } model.info_dict = save_dict torch.save(model, SAVE_NAME) if epoch in SAVE_EPOCH_LIST: if os.path.isfile(SAVE_NAME): shutil.copyfile( SAVE_NAME, SAVE_NAME.replace(".pt", "_{}.pt".format(epoch))) writer.close()
def train(opt): # Deal with feature things before anything opt.use_fc, opt.use_att = utils.if_use_feat(opt.caption_model) if opt.use_box: opt.att_feat_size = opt.att_feat_size + 5 loader = dataloader.DataLoader(opt) opt.vocab_size = loader.vocab_size opt.seq_length = loader.seq_length tb_summary_writer = tb and tb.SummaryWriter(opt.checkpoint_path) infos = {} histories = {} if opt.start_from is not None: # open old infos and check if models are compatible with open(os.path.join(opt.start_from, 'infos_'+opt.id+'.pkl'), 'rb') as f: infos = utils.pickle_load(f) saved_model_opt = infos['opt'] need_be_same = ["caption_model", "rnn_type", "rnn_size", "num_layers"] for checkme in need_be_same: assert vars(saved_model_opt)[checkme] == vars(opt)[ checkme], "Command line argument and saved model disagree on '%s' " % checkme if os.path.isfile(os.path.join(opt.start_from, 'histories_'+opt.id+'.pkl')): with open(os.path.join(opt.start_from, 'histories_'+opt.id+'.pkl'), 'rb') as f: histories = utils.pickle_load(f) else: infos['iter'] = 0 infos['epoch'] = 0 infos['iterators'] = loader.iterators infos['split_ix'] = loader.split_ix infos['vocab'] = loader.get_vocab() infos['opt'] = opt iteration = infos.get('iter', 0) epoch = infos.get('epoch', 0) val_result_history = histories.get('val_result_history', {}) loss_history = histories.get('loss_history', {}) lr_history = histories.get('lr_history', {}) ss_prob_history = histories.get('ss_prob_history', {}) loader.iterators = infos.get('iterators', loader.iterators) loader.split_ix = infos.get('split_ix', loader.split_ix) if opt.load_best_score == 1: best_val_score = infos.get('best_val_score', None) opt.vocab = loader.get_vocab() # DataParallel model = models.setup(opt).cuda() del opt.vocab dp_model = torch.nn.DataParallel(model) lw_model = LossWrapper(model, opt) dp_lw_model = torch.nn.DataParallel(lw_model) # not DataParallel # dp_model = models.setup(opt).cuda() # model = dp_model # del opt.vocab # dp_lw_model = LossWrapper(dp_model, opt) # lw_model = dp_lw_model epoch_done = True # Assure in training mode dp_lw_model.train() if opt.noamopt: assert opt.caption_model == 'transformer', 'noamopt can only work with transformer' optimizer = utils.get_std_opt( model, factor=opt.noamopt_factor, warmup=opt.noamopt_warmup) optimizer._step = iteration elif opt.reduce_on_plateau: optimizer = utils.build_optimizer(model.parameters(), opt) optimizer = utils.ReduceLROnPlateau(optimizer, factor=0.5, patience=3) else: optimizer = utils.build_optimizer(model.parameters(), opt) # Load the optimizer if vars(opt).get('start_from', None) is not None and os.path.isfile(os.path.join(opt.start_from, "optimizer.pth")): optimizer.load_state_dict(torch.load( os.path.join(opt.start_from, 'optimizer.pth'))) def save_checkpoint(model, infos, optimizer, histories=None, append=''): if len(append) > 0: append = '-' + append # if checkpoint_path doesn't exist if not os.path.isdir(opt.checkpoint_path): os.makedirs(opt.checkpoint_path) checkpoint_path = os.path.join( opt.checkpoint_path, 'model%s.pth' % (append)) torch.save(model.state_dict(), checkpoint_path) print("model saved to {}".format(checkpoint_path)) optimizer_path = os.path.join( opt.checkpoint_path, 'optimizer%s.pth' % (append)) torch.save(optimizer.state_dict(), optimizer_path) with open(os.path.join(opt.checkpoint_path, 'infos_'+opt.id+'%s.pkl' % (append)), 'wb') as f: utils.pickle_dump(infos, f) if histories: with open(os.path.join(opt.checkpoint_path, 'histories_'+opt.id+'%s.pkl' % (append)), 'wb') as f: utils.pickle_dump(histories, f) try: while True: if epoch_done: if not opt.noamopt and not opt.reduce_on_plateau: # Assign the learning rate if epoch > opt.learning_rate_decay_start and opt.learning_rate_decay_start >= 0: frac = ( epoch - opt.learning_rate_decay_start) // opt.learning_rate_decay_every decay_factor = opt.learning_rate_decay_rate ** frac opt.current_lr = opt.learning_rate * decay_factor else: opt.current_lr = opt.learning_rate # set the decayed rate utils.set_lr(optimizer, opt.current_lr) # Assign the scheduled sampling prob if epoch > opt.scheduled_sampling_start and opt.scheduled_sampling_start >= 0: frac = ( epoch - opt.scheduled_sampling_start) // opt.scheduled_sampling_increase_every opt.ss_prob = min( opt.scheduled_sampling_increase_prob * frac, opt.scheduled_sampling_max_prob) model.ss_prob = opt.ss_prob # If start self critical training if opt.self_critical_after != -1 and epoch >= opt.self_critical_after: sc_flag = True init_scorer(opt.cached_tokens) else: sc_flag = False epoch_done = False start = time.time() # Load data from train split (0) data = loader.get_batch('train') print('Read data:', time.time() - start) torch.cuda.synchronize() start = time.time() tmp = [data['fc_feats'], data['att_feats'], data['labels'], data['masks'], data['att_masks']] tmp = [_ if _ is None else _.cuda() for _ in tmp] fc_feats, att_feats, labels, masks, att_masks = tmp optimizer.zero_grad() model_out = dp_lw_model(fc_feats, att_feats, labels, masks, att_masks, data['gts'], torch.arange(0, len(data['gts'])), sc_flag) loss = model_out['loss'].mean() loss.backward() utils.clip_gradient(optimizer, opt.grad_clip) optimizer.step() train_loss = loss.item() torch.cuda.synchronize() end = time.time() if not sc_flag: print("iter {} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}" .format(iteration, epoch, train_loss, end - start)) else: print("iter {} (epoch {}), avg_reward = {:.3f}, time/batch = {:.3f}" .format(iteration, epoch, model_out['reward'].mean(), end - start)) # Update the iteration and epoch iteration += 1 if data['bounds']['wrapped']: epoch += 1 epoch_done = True # Write the training loss summary if (iteration % opt.losses_log_every == 0): add_summary_value(tb_summary_writer, 'train_loss', train_loss, iteration) if opt.noamopt: opt.current_lr = optimizer.rate() elif opt.reduce_on_plateau: opt.current_lr = optimizer.current_lr add_summary_value(tb_summary_writer, 'learning_rate', opt.current_lr, iteration) add_summary_value( tb_summary_writer, 'scheduled_sampling_prob', model.ss_prob, iteration) if sc_flag: add_summary_value( tb_summary_writer, 'avg_reward', model_out['reward'].mean(), iteration) loss_history[iteration] = train_loss if not sc_flag else model_out['reward'].mean( ) lr_history[iteration] = opt.current_lr ss_prob_history[iteration] = model.ss_prob # update infos infos['iter'] = iteration infos['epoch'] = epoch infos['iterators'] = loader.iterators infos['split_ix'] = loader.split_ix # make evaluation on validation set, and save model if (iteration % opt.save_checkpoint_every == 0): # eval model eval_kwargs = {'split': 'val', 'dataset': opt.input_json} eval_kwargs.update(vars(opt)) val_loss, predictions, lang_stats = eval_utils.eval_split( dp_model, lw_model.crit, loader, eval_kwargs) if opt.reduce_on_plateau: if 'CIDEr' in lang_stats: optimizer.scheduler_step(-lang_stats['CIDEr']) else: optimizer.scheduler_step(val_loss) # Write validation result into summary add_summary_value(tb_summary_writer, 'validation loss', val_loss, iteration) if lang_stats is not None: for k, v in lang_stats.items(): add_summary_value(tb_summary_writer, k, v, iteration) val_result_history[iteration] = { 'loss': val_loss, 'lang_stats': lang_stats, 'predictions': predictions} # Save model if is improving on validation result if opt.language_eval == 1: current_score = lang_stats['CIDEr'] else: current_score = - val_loss best_flag = False if best_val_score is None or current_score > best_val_score: best_val_score = current_score best_flag = True # Dump miscalleous informations infos['best_val_score'] = best_val_score histories['val_result_history'] = val_result_history histories['loss_history'] = loss_history histories['lr_history'] = lr_history histories['ss_prob_history'] = ss_prob_history save_checkpoint(model, infos, optimizer, histories) if opt.save_history_ckpt: save_checkpoint(model, infos, optimizer, append=str(iteration)) if best_flag: save_checkpoint(model, infos, optimizer, append='best') # Stop if reaching max epochs if epoch >= opt.max_epochs and opt.max_epochs != -1: break except (RuntimeError, KeyboardInterrupt): print('Save ckpt on exception ...') save_checkpoint(model, infos, optimizer) print('Save ckpt done.') stack_trace = traceback.format_exc() print(stack_trace)
import dataloader import torch import numpy as np base = r'C:\Users\shaha\Documents\DeepGit\DeepLearningSeminar\project\dataraw\train-clean-100' batch_size = 1 T = 4 target_sr = 8e3 ds_train = dataloader.AudioDataset(path=base, batch_size=batch_size, T=T, target_sr=target_sr) dl_train = dataloader.DataLoader(dataset=ds_train, batch_size=batch_size) len_t = len(ds_train) for idx, (x, y) in enumerate(dl_train): data = (x, y) torch.save( data, r'C:\Users\shaha\Documents\DeepGit\DeepLearningSeminar\project\data\{}.pt' .format(idx)) precentile_done = round(100 * (idx + 1) / len_t) progress_symbols = int(np.floor(precentile_done * 80 / 100)) print('\r[' + ('#') * progress_symbols + (' ') * (80 - progress_symbols) + ']' + 'progress {}/100%'.format(precentile_done), end='') print('*' * 80) print('finished !')
import os os.environ["CUDA_VISIBLE_DEVICES"] = "1" import sys from models import attgan from tensorflow.keras import losses, optimizers, metrics import numpy as np import tensorflow as tf import dataloader import utils from settings import * train_dloader = dataloader.DataLoader("train", BATCH_SIZE) valid_dloader = dataloader.DataLoader("valid", BATCH_SIZE) test_dloader = dataloader.DataLoader("test", BATCH_SIZE) model = attgan.AttGAN() # criterion_reconstruction = losses.MeanAbsoluteError() # criterion_adversarial = losses.BinaryCrossentropy() # criterion_attribute = losses.BinaryCrossentropy() def criterion_MAE(y_true, y_pred): n = y_true.shape[0] loss = tf.math.abs(y_true - y_pred) loss = tf.reshape(loss, shape=(n, -1)) loss = tf.reduce_sum(loss, axis=-1)
def train(model_dict=None, using_cuda=True, learning_rate=0.06,\ momentum=0.3, batch_size=32, epochs=5, coef=1.0, interval=10): """ training procedure Args: If model_dict is given (a file address), it will continue training on the given model. Otherwise, it would train a new model from scratch. If using_cuda is true, the training would be conducted on GPU. Learning_rate and momentum is for SGD optimizer. coef is the coefficent between the cross-entropy loss and the penalization term. interval is the frequncy of reporting. the result will be saved with a form "model_dict_+current time", which could be used for further training """ if using_cuda: net = Net().cuda() else: net = Net() if model_dict != None: net.load_state_dict(torch.load(model_dict)) optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=momentum) criterion = nn.CrossEntropyLoss() dataset = dataloader.DataLoader("train_set.pkl", batch_size, using_cuda=using_cuda) #statistics loss_count = 0 prepare_time = 0 run_time = 0 count = 0 for epoch in range(epochs): print("epoch: %d" % (epoch)) for i, batch in enumerate(dataset): t1 = time.time() X = batch["feature"] y = batch["class"] t2 = time.time() y_pred, y_penl = net(X) loss = criterion(y_pred, y) + torch.sum(y_penl) / batch_size * coef optimizer.zero_grad() loss.backward() nn.utils.clip_grad_norm(net.parameters(), 0.5) optimizer.step() t3 = time.time() loss_count += torch.sum(y_penl).data[0] prepare_time += (t2 - t1) run_time += (t3 - t2) p, idx = torch.max(y_pred.data, dim=1) count += torch.sum(torch.eq(idx.cpu(), y.data.cpu())) if (i + 1) % interval == 0: print("epoch : %d, iters: %d" % (epoch, i + 1)) print("loss count:" + str(loss_count / (interval * batch_size))) print("acuracy:" + str(count / (interval * batch_size))) print("penalty:" + str(torch.sum(y_penl).data[0] / batch_size)) print("prepare time:" + str(prepare_time)) print("run time:" + str(run_time)) prepare_time = 0 run_time = 0 loss_count = 0 count = 0 string = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()) torch.save(net.state_dict(), "model_dict_%s.dict" % (string))
def pipeline(args): ### Experiment setting update_feature = args.update_feature resume_epoch = args.resume_epoch lr = args.lr exp_name = 'RNN' num_epoch = 15 save_period = 200 log_period = 50 lr_scheduler_period = 5 lr_decay = 0.5 batchsize = 256 num_tf_thread = 8 clip_grad = 5.0 ### step1 : prepare data data = feature.PreprocessingRawdata(update_feature=update_feature) label = feature.GetLabels(data) data_standardized = feature.Standardize(data) data_train, data_validation, data_test, label_train, label_validation, label_test = \ feature.SplitData(data_standardized, label) # Get data iterator training_loader = dataloader.DataLoader(data=data_train, label=label_train, batchsize=batchsize, time=cfg.time.train_timeslots, mode='train') validation_loader = dataloader.DataLoader( data=data_validation, label=label_validation, batchsize=1, time=cfg.time.validation_timeslots, mode='validation') testing_loader = dataloader.DataLoader(data=data_test, label=label_test, batchsize=1, time=cfg.time.test_timeslots, mode='test') ### step2 : training # files exp_dir = os.path.join(exp_name) if os.path.exists(exp_dir) is False: os.makedirs(exp_dir) # Get Graph logging.info('Building Computational Graph...') # Get input shapes feature_name = data.minor_axis shapes = {} input_nodes = list(cfg.model.link.keys()) + ['weather'] for key in input_nodes: shapes[key] = len( [item for item in feature_name if item.startswith(key)]) # Build Graph prediction, loss, metric, label_sym = model.Build(shapes) # Optimizer learning_rate = tf.placeholder(shape=[], dtype=tf.float32, name='learning_rate') # optimizer = tf.train.GradientDescentOptimizer(learning_rate) # Clip gradient optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(loss) # gvs = optimizer.compute_gradients(loss) # capped_gvs = [(tf.clip_by_value(grad, -clip_grad, clip_grad), var) for grad, var in gvs] # optimizer = optimizer.apply_gradients(capped_gvs) # create session ans saver sess = tf.Session(config=tf.ConfigProto( intra_op_parallelism_threads=num_tf_thread)) # saver = tf.train.Saver() # Build the summary operation and summary writer # Training_MAPE = tf.placeholder(shape=[], dtype=tf.float32, name='Training_MAPE') # Validation_MAPE = tf.placeholder(shape=[], dtype=tf.float32, name='Validation_MAPE') # training_summary = tf.summary.scalar("Training_MAPE", Training_MAPE) # validation_summary = tf.summary.scalar("Validation_MAPE", Validation_MAPE) # learning_rate_summary = tf.summary.scalar("Learning_rate", learning_rate) # summary_writer = tf.summary.FileWriter(exp_dir, sess.graph) # Model params if resume_epoch == 0: # initializing init = tf.global_variables_initializer() logging.info('Initializing params...') sess.run(init) else: ## Loading logging.info('Loading the model of epoch[{}]...'.format(resume_epoch)) saver.restore(sess, exp_dir + '/model-{}'.format(resume_epoch)) #training logging.info("Starting training...") for epoch in range(resume_epoch + 1, num_epoch + 1): # Reset loader and metric training_loader.reset() validation_loader.reset() error_training = np.zeros((36)) count_training = np.zeros((36)) error_validation = np.zeros((36)) count_validation = np.zeros((36)) tic = time.time() # Training for batch in training_loader: # concat data and label data = batch.data data.update(batch.label) # print (data['loss_scale:0'].shape, data['loss_scale:0'].dtype) data['learning_rate:0'] = lr data['is_training:0'] = True # Feed data into graph _, error, label_batch = sess.run([optimizer, metric, label_sym], feed_dict=data) mask = (label_batch == label_batch) # Update metric error_training = error_training + error.sum(0) count_training += mask.sum(0) toc = time.time() # validation for batch in validation_loader: # concat data and label data = batch.data data.update(batch.label) data['is_training:0'] = False # Feed data into graph error, label_batch = sess.run([metric, label_sym], feed_dict=data) mask = (label_batch == label_batch) # Update metric error_validation = error_validation + error.sum(0) count_validation += mask.sum(0) # Speend and Error logging.info( "Epoch[{}] Speed:{:.2f} samples/sec Training MAPE={:.5f} Validation_MAPE={:.5f}" .format(epoch, training_loader.data_num / (toc - tic), error_training.sum() / count_training.sum(), error_validation.sum() / count_validation.sum())) print('training', (error_training / count_training).reshape(6, 6)) print('validation', (error_validation / count_validation).reshape(6, 6)) # Summary # if (epoch % log_period == 0): # train_summ, validation_summ, lr_summ = sess.run([training_summary, # validation_summary, # learning_rate_summary], # feed_dict={'Training_MAPE:0' : error_training.mean(), # 'Validation_MAPE:0' : error_validation.mean(), # 'learning_rate:0' : lr}) # summary_writer.add_summary(train_summ, epoch) # summary_writer.add_summary(validation_summ, epoch) # summary_writer.add_summary(lr_summ, epoch) # Save checkpoint # if (epoch % save_period == 0): # logging.info("Saving model of Epoch[{}]...".format(epoch)) # saver.save(sess, exp_dir + '/model', global_step=epoch) # Learning rate schedule if (epoch % lr_scheduler_period == 0): lr *= lr_decay logging.info("Optimization Finished!") # Prediction keys = list(prediction.keys()) keys.sort() prediction = [prediction[key] for key in keys] traveltime_result = [] for batch in testing_loader: data = batch.data data['is_training:0'] = False aux = batch.aux # Feed data into graph pred = sess.run(prediction, feed_dict=data) for index, key in enumerate(keys): intersection, tollgate = key.split('_') tollgate = tollgate[-1] time_now = cfg.time.test_timeslots[aux + 6:aux + 12] for i in range(6): avg_time = pred[index][0][i] left = datetime.strptime(time_now[i], "%Y-%m-%d %H:%M:%S") right = left + timedelta(minutes=cfg.time.time_interval) item = dict(intersection_id=intersection, tollgate_id=tollgate, time_window='[{},{})'.format(left, right), avg_travel_time=avg_time) traveltime_result.append(item) # save prediction traveltime_result = pd.DataFrame(traveltime_result, columns=[ 'intersection_id', 'tollgate_id', 'time_window', 'avg_travel_time' ]) traveltime_result.to_csv(os.path.join( cfg.data.prediction_dir, '{}_travelTime.csv'.format(exp_name)), sep=',', header=True, index=False) logging.info('Prediction Finished!') sess.close()