def crossValidateRMAE(): """使用交叉验证验证模型精度""" logger.info("{}-crossValidateRMAE-{}".format('*' * 25, '*' * 25)) X, y = getTrainData() _, trainModels = model.getModel() stackModel = model.stacking(trainModels, LinearRegression()) rmae = tool.crossValueScore(stackModel, X, y, tool.computeRMAE) logger.info("stacking model, cross validate RMAE: {}".format(rmae)) names, models = model.getModel() for n, m in zip(names, models): rmae = tool.crossValueScore(m, X, y, tool.computeRMAE) logger.info("model: {}, cross validate RMAE: {}".format(n, rmae))
def setUpModel(): tf.reset_default_graph() model = getModel(words_data['input_tensor_length'], words_data['output_tensor_length']) model = tflearn.DNN(model, tensorboard_dir='tflearn_logs') model.load(MODEL_FILE) return model
def submit_file(): if request.method == 'POST': if 'file' not in request.files: flash('No file folder') return redirect(request.url) file = request.files['file'] if file.filename == '': flash('No file selected for uploading') return redirect(request.url) if file: filename = secure_filename(file.filename) # Security essential !! if os.path.exists( os.path.join(app.config['UPLOAD_FOLDER'], filename)): imageType = magic.from_file('uploads/' + filename, mime=True) filename = get_random_string(3) + '_' + (filename.split( '.', 1))[0] + '.' + (imageType.split('/'))[1] file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) model = getModel() result = getPrediction(filename, model) if result == '1.0': flash("Normal") elif result == '0.0': flash("Covid") else: flash("Undefined") flash(filename) send_async('prediction', filename=filename) return redirect('/')
def trainBySingleModel(): """训练单个模型并保存结果""" X, Y = getTrainData() names, models = model.getModel() for n, m in zip(names, models): singleModle = model.train_by_model(n, X, Y) io.saveData(singleModle, singleModelSaving.format(n))
def old1(): modl = model.getModel((150, 150, 3)) train_path, val_path, TestPath = pth.getPaths() train_generator, val_generator, test_generator = data_aug.getGens( train_path, val_path, TestPath, (150, 150)) hist = modl.fit_generator(train_generator, steps_per_epoch=1500 // 20, epochs=5, validation_data=val_generator, validation_steps=500 // 20).history
def main(): now = datetime.datetime.now() logger = Logger(opt.saveDir + '/logs_{}'.format(now.isoformat())) model, optimizer = getModel(opt) criterion = torch.nn.MSELoss() if opt.GPU > -1: print('Using GPU', opt.GPU) model = model.cuda(opt.GPU) criterion = criterion.cuda(opt.GPU) val_loader = torch.utils.data.DataLoader( Dataset(opt, 'val'), batch_size=1, shuffle=True if opt.DEBUG > 1 else False, num_workers=1) if opt.test: _, preds = val(0, opt, val_loader, model, criterion) torch.save({ 'opt': opt, 'preds': preds }, os.path.join(opt.saveDir, 'preds.pth')) return train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'), batch_size=opt.trainBatch, shuffle=True, num_workers=int(opt.nThreads)) for epoch in range(1, opt.nEpochs + 1): mark = epoch if opt.saveAllModels else 'last' log_dict_train, _ = train(epoch, opt, train_loader, model, criterion, optimizer) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if epoch % opt.valIntervals == 0: log_dict_val, preds = val(epoch, opt, val_loader, model, criterion) for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) saveModel( os.path.join(opt.saveDir, 'model_{}.checkpoint'.format(mark)), model) # optimizer logger.write('\n') if epoch % opt.dropLR == 0: lr = opt.LR * (0.1**(epoch // opt.dropLR)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr logger.close() torch.save(model.cpu(), os.path.join(opt.saveDir, 'model_cpu.pth'))
def getModel(): try: mod = tf.keras.models.load_model('../../data/models/' + args.model_name) print("Model " + args.model_name + " loaded.") except Exception: print("Creating new model.") mod = model.getModel("smallVGG") print() print(mod.summary()) return mod
def testSingleModelRMAE(): """使用所有数据进行训练,然后对训练数据进行预测""" logger.info("{}-testSingleModelRMAE-{}".format('*' * 25, '*' * 25)) X, Y = getTrainData() names, models = model.getModel() for n in names: m = io.getData(singleModelSaving.format(n)) pdtValue = m.predict(X) retMSE = tool.computeRMAE(Y, pdtValue) logger.info("model: {}, using all data, RMAE : {}".format(n, retMSE))
def main(): opt = opts().parse() now = datetime.datetime.now() logger = Logger(opt.saveDir, now.isoformat()) model, optimizer = getModel(opt) criterion = torch.nn.MSELoss().cuda() # if opt.GPU > -1: # print('Using GPU {}',format(opt.GPU)) # model = model.cuda(opt.GPU) # criterion = criterion.cuda(opt.GPU) # dev = opt.device model = model.cuda() val_loader = torch.utils.data.DataLoader( MPII(opt, 'val'), batch_size = 1, shuffle = False, num_workers = int(ref.nThreads) ) if opt.test: log_dict_train, preds = val(0, opt, val_loader, model, criterion) sio.savemat(os.path.join(opt.saveDir, 'preds.mat'), mdict = {'preds': preds}) return # pyramidnet pretrain一次,先定义gen的训练数据loader train_loader = torch.utils.data.DataLoader( MPII(opt, 'train'), batch_size = opt.trainBatch, shuffle = True if opt.DEBUG == 0 else False, num_workers = int(ref.nThreads) ) # 调用train方法 for epoch in range(1, opt.nEpochs + 1): log_dict_train, _ = train(epoch, opt, train_loader, model, criterion, optimizer) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if epoch % opt.valIntervals == 0: log_dict_val, preds = val(epoch, opt, val_loader, model, criterion) for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) #saveModel(model, optimizer, os.path.join(opt.saveDir, 'model_{}.checkpoint'.format(epoch))) torch.save(model, os.path.join(opt.saveDir, 'model_{}.pth'.format(epoch))) sio.savemat(os.path.join(opt.saveDir, 'preds_{}.mat'.format(epoch)), mdict = {'preds': preds}) logger.write('\n') if epoch % opt.dropLR == 0: lr = opt.LR * (0.1 ** (epoch // opt.dropLR)) print('Drop LR to {}'.format(lr)) adjust_learning_rate(optimizer, lr) logger.close() torch.save(model.cpu(), os.path.join(opt.saveDir, 'model_cpu.pth'))
def removePhone(phones,manufacturer,model): ''' in - phones- list of all phones manufacturer - the manufacturer of the phone we want to be removed model - the model of the phone we want to be removed out - True if it removed said phone False if said phone doesnt exist ''' for index in range (0,len(phones)): if getManufacturer(phones[index]) == manufacturer and getModel(phones[index]) == model: del phones[index] return True return False
def increaseAmount(phones,manufacturer,model,amount): ''' in - phones- list of all phones manufacturer - the manufacturer of the phone we want to be removed model - the model of the phone we want to be removed amount - the price amount we want to add to the phone <manufacturer,model> out - True if the modification was made False otherwise, if no such phone exists ''' for index in range (0,len(phones)): if getManufacturer(phones[index]) == manufacturer and getModel(phones[index]) == model: oldPrice = getPrice(phones[index]) setPrice(phones[index],oldPrice+amount) return True return False
def __init__(self, cfg): self.cfg = cfg os.makedirs(self.cfg.EXP.PATH, exist_ok=True) os.makedirs(self.cfg.EXP.PATH + '/valimg', exist_ok=True) # logger self.writer = SummaryWriter(self.cfg.EXP.PATH) # 计时器 self.t = {'iter': Timer(), 'train': Timer(), 'val': Timer()} # 保存实验环境 # TODO: 启用 temp = os.path.join(self.cfg.EXP.PATH, 'code') utils.copy_cur_env('./', temp, exception='exp') # 读取数据集 self.meanImg, self.trainloader, self.valloader = datasets.getData( self.cfg) # 定义网络 self.net = model.getModel(cfg) # 损失函数 self.criterion = torch.nn.MSELoss() # 优化器 self.optimizer = torch.optim.Adam( self.net.parameters(), lr=self.cfg.TRAIN.LR, weight_decay=self.cfg.TRAIN.WEIGHT_DECAY) # 初始化一些变量 self.beginEpoch = 1 self.batch = 1 self.bestacc = 0 # 载入预训练模型 if self.cfg.TRAIN.RESUME: print('Loading Model..........') saved_state = torch.load(self.cfg.TRAIN.RESUME_PATH) self.net.load_state_dict(saved_state['weights']) self.beginEpoch = saved_state['epoch'] self.batch = saved_state['batch'] self.bestacc = saved_state['bestacc'] # GPU设定 self.gpu = torch.cuda.is_available() and self.cfg.TRAIN.USE_GPU self.device = 'cuda' if self.gpu else 'cpu' if self.gpu: torch.cuda.set_device(self.cfg.TRAIN.GPU_ID[0]) self.criterion.cuda() if len(self.cfg.TRAIN.GPU_ID) > 1: self.net = torch.nn.DataParallel( self.net, device_ids=self.cfg.TRAIN.GPU_ID) self.net = self.net.cuda() else: self.net = self.net.cpu() self.criterion.cpu()
def train(audio_path, plot_matrix=False): x_data, y_data = get_set(26, 9, audio_path) x_data = keras.preprocessing.sequence.pad_sequences(x_data, maxlen=100) x_train, x_test, Y_train, Y_test = train_test_split(x_data, y_data, test_size=0.1, random_state=42) y_train = keras.utils.to_categorical(Y_train, 16) y_test = keras.utils.to_categorical(Y_test, 16) model = getModel((x_train.shape[1], x_train.shape[2]), y_train.shape[1]) history = model.fit(x_train, y_train, batch_size=10, epochs=137, verbose=1, validation_data=(x_test, y_test)) # Plot training & validation accuracy values plt.plot(history.history['acc']) plt.plot(history.history['val_acc']) plt.title('Model accuracy') plt.ylabel('Accuracy') plt.xlabel('Epoch') plt.legend(['Train', 'Test'], loc='upper left') plt.show() # Plot training & validation loss values plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('Model loss') plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['Train', 'Test'], loc='upper left') plt.show() score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) model.save(model_name) if plot_matrix: plot(x_test, Y_test, model_name)
def train(args): epochs = int(os.path.expanduser(args.epochs)) batch_size = int(os.path.expanduser(args.batch_size)) #TODO: # Add TensorBoard # Code Model # code f*****g loss f checkPointPath = './pretrainedModel/' if not os.path.isdir(checkPointPath): os.makedirs(checkPointPath) modelPath = os.path.join(checkPointPath, 'LastModel.hdf5') checkPoint = ModelCheckpoint( modelPath, monitor="val_loss", verbose=0, save_best_only=True, save_weights_only=False, mode="auto", save_freq="epoch", ) input_shape = (448, 448, 3) inputs = Input(input_shape) if os.path.exists(modelPath): model = load_model(modelPath) else: model = getModel(inputs) model.compile(loss=getloss, optimizer='adam') train_generator = YoloSequenceData('train', batch_size) validation_generator = YoloSequenceData('val', batch_size) model.fit_generator(train_generator, steps_per_epoch=len(train_generator), epochs=epochs, validation_data=validation_generator, validation_steps=len(validation_generator), callbacks=[checkPoint], verbose=0)
def main(): data = load_dataCSV() look_back = 28 jump=4 train_data, test_data = dp.rescale_data(data) trainX, trainY = dp.create_dataset(train_data, look_back) trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1])) testX, testY = dp.create_dataset(test_data, look_back) model = mod.getModel(look_back) model.fit( trainX, trainY, batch_size=128, nb_epoch=300, validation_split=0.10) pred,perfs=mod.testModel(model,testX,testY,jump,look_back) actual_test_data=test_data[len(test_data)-len(pred):] print("\n Average Covarance between predicted and actual prices on only predicted days:") print(np.mean(perfs)) print("\n Covarance between predicted and actual prices on all days:") print(np.cov(actual_test_data,pred)[1][0]) plt.figure(3) plt.plot(actual_test_data) plt.figure(4) plt.plot(pred) mod.saveModel(model,'lstm3')
def compileModel(epochs=5000, seve_name='model'): (X, Y, urls) = getDataSet() model = getModel() trainModel(model, X, Y, epochs) y_pred = [x[0] for x in model.predict(X)] accuracy, false_like, false_dislike = calculateModelMetrics(y_pred, Y) corelation = getCorrelation(y_pred, Y) realTolerance = getMid(Y) binTolerance = getMid([int(x > 0.5) for x in y_pred]) modelTolerance = getMid(y_pred) new_data = [{ 'prediction': (Mark.like.name if x > modelTolerance else Mark.dislike.name) } for x in y_pred] memo.upd_urls(dict(zip(urls, new_data))) metrics = { 'real tolerance': realTolerance, 'bin tolerance': binTolerance, 'model tolerance': modelTolerance, 'accuracy': accuracy, 'correlation': corelation, 'false_like': false_like, 'false_dislike': false_dislike, } print("{:<16} {:<8}".format('METRIC', 'VALUE')) for key, val in metrics.items(): print("{:<16} {:<8}".format(key, val)) if not os.path.isdir(ROOT_PATH + '/models/'): os.mkdir(ROOT_PATH + '/models/') model.save('models/' + seve_name + '.h5')
noise_level=noise) train_dataloader = DataLoader(train_dataset, config.NNBATCHSIZE, shuffle=True, num_workers=16) valid_dataset = IronDataset(train[val_index], train_tr[val_index], seq_len=config.GROUP_BATCH_SIZE, flip=False) valid_dataloader = DataLoader(valid_dataset, config.NNBATCHSIZE, shuffle=False) it = 0 model = getModel(config) early_stopping = EarlyStopping( patience=10, is_maximize=True, checkpoint_path=os.path.join( config.outdir, "gru_clean_checkpoint_expid_{}_fold_{}_iter_{}.pt".format( config.expriment_id, index, it))) weight = None # cal_weights() criterion = nn.CrossEntropyLoss(weight=weight) optimizer = torch.optim.Adam(model.parameters(), lr=config.LR) optimizer = torchcontrib.optim.SWA(optimizer, swa_start=10, swa_freq=2,
if __name__ == '__main__': global C C = Conf() res = [] df = getData(C.STRFRQ[C.DATAFREQ]) # df = prepareData(df) ndf = normaliz(df) dfY = df[C.Y] for sn in SCENARIOS: #encoder.add(Dropout(0.5)) #encoder.add(LSTM(output_dim=C.XOUT_DIM, return_sequences=True, stateful=True)) C.overwrite(sn) print(" - " + C.SCENARIO) train_Xs, train_Y = getTrainData(C, ndf, dfY) val_Xs, val_Y = getTrainData(C, ndf, dfY, 'test') model = getModel(C) #cost = model.train_on_batch([train_Xs[i][0:C.BATCH_SIZE] for i in range(len(train_Xs))], train_Y[0:C.BATCH_SIZE]) #print(cost) re = model.fit(train_Xs, train_Y, batch_size=C.BATCH_SIZE, nb_epoch=20, validation_data=(val_Xs, val_Y)) res.append([C.SCENARIO, re.history]) ResAnaysis(res)
# 发送方向键 if bestDir == 0: bodyElem.send_keys(Keys.UP) elif bestDir == 1: bodyElem.send_keys(Keys.RIGHT) elif bestDir == 2: bodyElem.send_keys(Keys.DOWN) elif bestDir == 3: bodyElem.send_keys(Keys.LEFT) # 循环发送方向键 for i in range(10000): # 对当前画面建模 # 返回false时,说明程序运行过快,tile元素已不在原DOM中,搜索不到,所以跳过此次循环 if model.getModel(browser) == False: continue # 发送方向键 sendDirKey() time.sleep(0.02) # 检测游戏是否失败 try: if browser.find_element_by_class_name('game-over'): break except: None # 检测游戏是否胜利 try: if browser.find_element_by_class_name('game-won'): break except:
# Get all the word trie nodes that are asociated with our given prefix tries = [('', currentTrie)] if not prefix else list( currentTrie.getLetters(prefix[-1])) # Obtain every sentence from our data that matches the given prefix # and occurs more than frequency_threshold times results = dict( ((sentence, count) for _, tr in tries\ for sentence, count in tr if count > frequency_threshold) ) # Sort the resulting sentences by their frequencies and only keep the top max_results return sorted(results, key=results.__getitem__, reverse=True)[:max_results] if __name__ == '__main__': # Console interface for autocomplete datapath = os.path.join(os.path.dirname(__file__), 'sample_conversations.json') trie = getModel(datapath) print("Type in a prefix to autocomplete",\ "Use SET_MAX_RESULTS n to set the maximum number of results.",\ "Use SET_FREQ_THRESHOLD n to set the frequency threshold.",\ "Type QUIT to exit.") max_results, frequency_threshold = None, 0 while True: inp = input("prefix> ").strip() if inp == "QUIT": break elif inp[:16] == 'SET_MAX_RESULTS ': try: max_results = max(0, int(inp[16:])) except: print("Error setting max_results to", "'" + inp[16:] + "'.", 'n must be a nonnegative integer.')
def main(experiment, logging, augmentation, dataset, model, metric, training): if experiment["reproducible"]: print('fix seed on') seed = 0 random.seed(seed) # augmentation np.random.seed(seed) # numpy ia.seed(seed) # imgaug library torch.manual_seed(seed) # cpu torch.cuda.manual_seed(seed) # gpu torch.cuda.manual_seed_all(seed) # multi gpu torch.backends.cudnn.enabled = False # cudnn library torch.backends.cudnn.deterministic = True ################## # logging # ################## LOG = Logger(**logging) LOG('print', name='config', values=json.dumps(config)) ################## # dataset # ################## datasets = getDataset(**dataset, **augmentation) LOG('slack', name='dataset', values=[str(datasets['train']), str(datasets['val'])]) ################## # model # ################## MODEL = getModel(**model) input_size = [augmentation['channel']] + augmentation['size'] MODEL.modelSummary(input_size, LOG) ################## # metric # ################## metricParser = TypeParser(types={ "IOU": M.IOU, "DICE": M.DICE, "accuracy": M.Accuracy, "f1": M.F1, }) metrics = [metricParser(**m) for m in metric] ################## # training # ################## trainer = Trainer(model=MODEL, datasets=datasets, metrics=metrics, LOG=LOG) try: trainer.train(**config["training"]) except Exception as e: LOG('slack', name='warning', values='abrupt end, {}'.format(e)) LOG.finish() print('abrupt end, {}'.format(e)) print(traceback.format_exc()) infer = Inference( model=MODEL, datasets=datasets, LOG=LOG, metrics=metrics, visualizations=None, ) infer() LOG.finish()
import dataLoader import model from keras.models import model_from_json from keras.callbacks import TensorBoard import os #os.environ['CUDA_VISIBLE_DEVICES'] = '1' processedData, unprocessedData = dataLoader.fetch_data('data/') autoencoder = model.getModel(256, 256, 8) x_train = processedData x_test = processedData[1,:,:,:].reshape(1,256,256,8) autoencoder.fit(x_train, x_train, epochs=100000, batch_size=4, shuffle=True, validation_data=(x_test, x_test), callbacks=[TensorBoard(log_dir='conv_autoencoder')], verbose=2) autoencoder_json = autoencoder.to_json() with open("model.json", "w") as json_file: json_file.write(autoencoder_json) # serialize weights to HDF5 autoencoder.save_weights("model.h5") print("Saved model to disk") autoencoder.save('autoencoder.h5')
def main(): now = datetime.datetime.now() logger = Logger(args.save_path + '/logs_{}'.format(now.isoformat())) model = getModel(args) cudnn.benchmark = True optimizer = torch.optim.SGD(model.parameters(), args.LR, momentum=args.momentum, weight_decay=args.weight_decay) valSource_dataset = SourceDataset('test', ref.nValViews) valTarget_dataset = TargetDataset('test', ref.nValViews) valSource_loader = torch.utils.data.DataLoader(valSource_dataset, batch_size = 1, shuffle=False, num_workers=1, pin_memory=True, collate_fn=collate_fn_cat) valTarget_loader = torch.utils.data.DataLoader(valTarget_dataset, batch_size = 1, shuffle=False, num_workers=1, pin_memory=True, collate_fn=collate_fn_cat) if args.test: f = {} for split in splits: f['{}'.format(split)] = open('{}/{}.txt'.format(args.save_path, split), 'w') test(args, valSource_loader, model, None, f['valSource'], 'valSource') test(args, valTarget_loader, model, None, f['valTarget'], 'valTarget') return train_dataset = Fusion(SourceDataset, TargetDataset, nViews = args.nViews, targetRatio = args.targetRatio, totalTargetIm = args.totalTargetIm) trainTarget_dataset = train_dataset.targetDataset train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batchSize, shuffle=not args.test, num_workers=args.workers if not args.test else 1, pin_memory=True, collate_fn=collate_fn_cat) trainTarget_loader = torch.utils.data.DataLoader( trainTarget_dataset, batch_size=args.batchSize, shuffle=False, num_workers=args.workers if not args.test else 1, pin_memory=True, collate_fn=collate_fn_cat) M = None if args.shapeWeight > ref.eps: print 'getY...' Y = getY(train_dataset.sourceDataset) M = initLatent(trainTarget_loader, model, Y, nViews = args.nViews, S = args.sampleSource, AVG = args.AVG) print 'Start training...' for epoch in range(1, args.epochs + 1): adjust_learning_rate(optimizer, epoch, args.dropLR) train_mpjpe, train_loss, train_unSuploss = train(args, train_loader, model, optimizer, M, epoch) valSource_mpjpe, valSource_loss, valSource_unSuploss = validate(args, 'Source', valSource_loader, model, None, epoch) valTarget_mpjpe, valTarget_loss, valTarget_unSuploss = validate(args, 'Target', valTarget_loader, model, None, epoch) train_loader.dataset.targetDataset.shuffle() if args.shapeWeight > ref.eps and epoch % args.intervalUpdateM == 0: M = stepLatent(trainTarget_loader, model, M, Y, nViews = args.nViews, lamb = args.lamb, mu = args.mu, S = args.sampleSource) logger.write('{} {} {}\n'.format(train_mpjpe, valSource_mpjpe, valTarget_mpjpe)) logger.scalar_summary('train_mpjpe', train_mpjpe, epoch) logger.scalar_summary('valSource_mpjpe', valSource_mpjpe, epoch) logger.scalar_summary('valTarget_mpjpe', valTarget_mpjpe, epoch) logger.scalar_summary('train_loss', train_loss, epoch) logger.scalar_summary('valSource_loss', valSource_loss, epoch) logger.scalar_summary('valTatget_loss', valTarget_loss, epoch) logger.scalar_summary('train_unSuploss', train_unSuploss, epoch) logger.scalar_summary('valSource_unSuploss', valSource_unSuploss, epoch) logger.scalar_summary('valTarget_unSuploss', valTarget_unSuploss, epoch) if epoch % 10 == 0: torch.save({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer' : optimizer.state_dict(), }, args.save_path + '/checkpoint_{}.pth.tar'.format(epoch)) logger.close()
result_data.append([bag, class_row]) # print 'tf data = {}'.format(result_data) return result_data if __name__ == '__main__': classes, all_words, docs = parseData() training_data = prepareDataForTf(classes, all_words, docs) random.shuffle(training_data) training = np.array(training_data) train_x = list(training[:, 0]) train_y = list(training[:, 1]) # print ("shape = {}\nx = {}\ny = {}".format(training.shape, train_x, train_y)) tf.reset_default_graph() net = getModel(len(train_x[0]), len(train_y[0])) # Define model and setup tensorboard model = tflearn.DNN(net, tensorboard_dir=DATA_PATH + 'tflearn_logs') # Start training (apply gradient descent algorithm) model.fit(train_x, train_y, n_epoch=1000, batch_size=8, show_metric=False) model.save(MODEL_FILE) with open(WORDS_FILE, 'w') as f: pickle.dump( { 'words': all_words, 'classes': classes, 'input_tensor_length': len(train_x[0]), 'output_tensor_length': len(train_y[0]) }, f)
from config import config from soundProcessing import batchgen_and_numsamples from model import getModel from keras import callbacks import os bgen, numSamps = batchgen_and_numsamples() model = getModel() class SaveEpoch(callbacks.Callback): def on_epoch_end(self, epch, thrd = {}): model.save_weights(os.path.join(config["projectDir"], "savedModel_%d"%epch), overwrite=True) model.fit_generator(bgen, samples_per_epoch=numSamps, nb_epoch= int(config["epochs"]) , max_q_size = 100, callbacks=[SaveEpoch()] )
def main(): now = datetime.datetime.now() logger = Logger(args.save_path + '/logs_{}'.format(now.isoformat())) model = getModel(args) cudnn.benchmark = True optimizer = torch.optim.SGD(model.parameters(), args.LR, momentum=args.momentum, weight_decay=args.weight_decay) valSource_dataset = SourceDataset('test', ref.nValViews) valTarget_dataset = TargetDataset('test', ref.nValViews) valSource_loader = torch.utils.data.DataLoader(valSource_dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=True, collate_fn=collate_fn_cat) valTarget_loader = torch.utils.data.DataLoader(valTarget_dataset, batch_size=1, shuffle=False, num_workers=1, pin_memory=True, collate_fn=collate_fn_cat) if args.test: f = {} for split in splits: f['{}'.format(split)] = open( '{}/{}.txt'.format(args.save_path, split), 'w') test(args, valSource_loader, model, None, f['valSource'], 'valSource') test(args, valTarget_loader, model, None, f['valTarget'], 'valTarget') return train_dataset = Fusion(SourceDataset, TargetDataset, nViews=args.nViews, targetRatio=args.targetRatio, totalTargetIm=args.totalTargetIm) trainTarget_dataset = train_dataset.targetDataset train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batchSize, shuffle=not args.test, num_workers=args.workers if not args.test else 1, pin_memory=True, collate_fn=collate_fn_cat) trainTarget_loader = torch.utils.data.DataLoader( trainTarget_dataset, batch_size=args.batchSize, shuffle=False, num_workers=args.workers if not args.test else 1, pin_memory=True, collate_fn=collate_fn_cat) M = None if args.shapeWeight > ref.eps: print 'getY...' Y = getY(train_dataset.sourceDataset) M = initLatent(trainTarget_loader, model, Y, nViews=args.nViews, S=args.sampleSource, AVG=args.AVG) print 'Start training...' for epoch in range(1, args.epochs + 1): adjust_learning_rate(optimizer, epoch, args.dropLR) train_mpjpe, train_loss, train_unSuploss = train( args, train_loader, model, optimizer, M, epoch) valSource_mpjpe, valSource_loss, valSource_unSuploss = validate( args, 'Source', valSource_loader, model, None, epoch) valTarget_mpjpe, valTarget_loss, valTarget_unSuploss = validate( args, 'Target', valTarget_loader, model, None, epoch) train_loader.dataset.targetDataset.shuffle() if args.shapeWeight > ref.eps and epoch % args.intervalUpdateM == 0: M = stepLatent(trainTarget_loader, model, M, Y, nViews=args.nViews, lamb=args.lamb, mu=args.mu, S=args.sampleSource) logger.write('{} {} {}\n'.format(train_mpjpe, valSource_mpjpe, valTarget_mpjpe)) logger.scalar_summary('train_mpjpe', train_mpjpe, epoch) logger.scalar_summary('valSource_mpjpe', valSource_mpjpe, epoch) logger.scalar_summary('valTarget_mpjpe', valTarget_mpjpe, epoch) logger.scalar_summary('train_loss', train_loss, epoch) logger.scalar_summary('valSource_loss', valSource_loss, epoch) logger.scalar_summary('valTatget_loss', valTarget_loss, epoch) logger.scalar_summary('train_unSuploss', train_unSuploss, epoch) logger.scalar_summary('valSource_unSuploss', valSource_unSuploss, epoch) logger.scalar_summary('valTarget_unSuploss', valTarget_unSuploss, epoch) if epoch % 10 == 0: torch.save( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), }, args.save_path + '/checkpoint_{}.pth.tar'.format(epoch)) logger.close()
def train(train_batch, validate_batch, test_data, mask): # nb_train = len(train_data) x = tf.placeholder(tf.float32, shape=(None, 256, 256, 24), name='x_input') y_ = tf.placeholder(tf.float32, shape=(None, 256, 256, 24), name='y_label') x_k = tf.placeholder(tf.complex64, shape=(None, 256, 256, 12), name='x_kspace') mask_k = tf.placeholder(tf.complex64, shape=(None, 256, 256, 12), name='mask') features, labels, kx_mask, mask_c = setup_inputs(train_batch, mask, BATCH_SIZE) f_val, l_val, kx_val, m_val = setup_inputs(validate_batch, mask, BATCH_SIZE) y = getModel(x, x_k, mask_k) global_step = tf.Variable(0., trainable=False) with tf.name_scope('mse_loss'): total_loss = mae(y_, y) lr = tf.train.exponential_decay( lr_base, global_step=global_step, decay_steps=(nb_train_samples + BATCH_SIZE - 1) // BATCH_SIZE, decay_rate=lr_decay_rate, staircase=False) with tf.name_scope("train"): train_step = tf.train.AdamOptimizer(lr).minimize( total_loss, global_step=global_step) saver = tf.train.Saver() with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) for i in range(EPOCHS): loss_sum = 0.0 count_batch = 0 ave_loss = 0. nb_batches = int(np.ceil(nb_train_samples // BATCH_SIZE)) for n_batch in range(nb_batches): features_trian, labels_train, kx_mask_train, mask_c_train = sess.run( [features, labels, kx_mask, mask_c]) _, loss_value, step = sess.run( [train_step, total_loss, global_step], feed_dict={ x: features_trian, y_: labels_train, x_k: kx_mask_train, mask_k: mask_c_train }) loss_sum += loss_value count_batch += 1 ave_loss = loss_sum / count_batch loss['batch'].append(ave_loss) print('Epoch %3d-batch %3d/%3d training loss: %8f' % (i + 1, count_batch, nb_batches, ave_loss)) # evaluate if count_batch % eval_every == 0: count_batch_val = 0 loss_sum_val = 0. ave_loss_val = 0. loss['count'].append(ave_loss) nb_batches_val = int(np.ceil(nb_val // BATCH_SIZE)) for n_batch_val in range(nb_batches_val): features_val, labels_val, kx_mask_val, mask_c_val = sess.run( [f_val, l_val, kx_val, m_val]) loss_value_val, pred_val = sess.run( [total_loss, y], feed_dict={ x: features_val, y_: labels_val, x_k: kx_mask_val, mask_k: mask_c_val }) loss_sum_val += loss_value_val count_batch_val += 1 ave_loss_val = loss_sum_val / count_batch_val print('Epoch %3d-batch %3d/%3d validation loss: %8f' % (i + 1, count_batch_val, n_batch_val, ave_loss_val)) val_loss.append(ave_loss_val) loss['epoch'].append(ave_loss) saver.save(sess, os.path.join(model_save_path, model_name), global_step=global_step) # test every 5 epochs if (i + 1) % test_every == 0: count = 0 for y_test, n_batch in iterate_minibatch(test_data, batch_size=1, shuffle=False): features_test, labels_test, kx_mask_test, mask_c_test = setup_inputs_test( y_test, mask, norm=None) test_loss, prediction = sess.run( [total_loss, y], feed_dict={ x: features_test, y_: labels_test, x_k: kx_mask_test, mask_k: mask_c_test }) count += 1 print('The loss of NO. %2d test data is %.8f' % (count, test_loss)) pred_c = real2complex_array(prediction) pred = np.squeeze( np.sqrt(np.sum(np.square(np.abs(pred_c)), axis=-1))) plt.figure(1) plt.imshow(pred, cmap='gray') fig_name = os.path.join('./result', '%d_out_%d.png' % (count, i + 1)) plt.savefig(fig_name) coord.request_stop() coord.join(threads) loss_plot('batch') loss_plot('count') np.savetxt(os.path.join("./result", 'train_batch_loss.txt'), np.asarray(loss['batch'])) np.savetxt(os.path.join("./result", 'train_count_loss.txt'), np.asarray(loss['count'])) np.savetxt(os.path.join("./result", 'val_loss.txt'), np.asarray(val_loss))
import torch from torch import optim import PIL.Image import utils import model import cv2 STEPS = 2500 model = model.getModel() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') content_image = utils.load_image( 'https://www.rover.com/blog/wp-content/uploads/2019/01/6342530545_45ec8696c8_b-960x540.jpg' ).to(device) style_image = utils.load_image( 'https://images2.minutemediacdn.com/image/upload/c_crop,h_1595,w_2835,x_0,y_408/f_auto,q_auto,w_1100/v1556647489/shape/mentalfloss/62280-mona_lisa-wiki.jpg' ).to(device) # get content and style features only once before training content_features = utils.get_features(content_image, model) style_features = utils.get_features(style_image, model) # calculate the gram matrices for each layer of our style representation style_grams = { layer: utils.gram_matrix(style_features[layer]) for layer in style_features }
def setUpClass(self): datapath = os.path.join(os.path.dirname(__file__), 'data', 'test_conversations.json') self.model = getModel(datapath)
from keras import models, layers import numpy as np import sincnet from keras.layers import Dense, Dropout, Activation print('N_filt ' + str(cnn_N_filt)) print('N_filt len ' + str(cnn_len_filt)) print('FS ' + str(fs)) print('WLEN ' + str(wlen)) input_shape = (wlen, 1) out_dim = class_lay[0] from model import getModel model = getModel(input_shape, out_dim) optimizer = RMSprop(lr=lr, rho=0.9, epsilon=1e-8) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) checkpoints_path = os.path.join(output_folder, 'checkpoints') tb = TensorBoard(log_dir=os.path.join(output_folder, 'logs', 'SincNet')) checkpointer = ModelCheckpoint(filepath=os.path.join(checkpoints_path, 'SincNet.hdf5'), verbose=1, save_best_only=False) if not os.path.exists(checkpoints_path): os.makedirs(checkpoints_path)
def getSingleModel(): """获得单个模型的名称""" names, models = model.getModel() return names