def get_dataset(dataset_path, batch_size, strategy=None): # Load dataset dataset = preprocess.load_dataset(dataset_path) # Get max length max_mel_length = 0 max_token_length = 0 for b, (mel, token) in enumerate(dataset): max_mel_length = max(max_mel_length, mel.shape[0]) max_token_length = max(max_token_length, token.shape[0]) print("Max mel length :", max_mel_length) print("Max token length :", max_token_length) num_mels = mel.shape[1] # Pad for batching dataset = dataset.padded_batch( batch_size, padded_shapes=( [max_mel_length, num_mels], # mel_specs [max_token_length]) # labels ) # Prefetch dataset (cuda streaming) dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) if strategy is not None: dataset = strategy.experimental_distribute_dataset(dataset) return dataset
def main(argv): if len(argv) > 1: raise app.UsageError('Too many command-line arguments.') dataset = preprocessor.get_dataset( preprocessor.load_dataset(FLAGS.dataset_path), preprocessor.load_json(FLAGS.split_path)) preprocessor.write_dataset(dataset, FLAGS.save_path) token_vocab = preprocessor.get_token_vocab(FLAGS.save_path) preprocessor.write_token_vocab(token_vocab, FLAGS.save_path)
def main(cfg): with Experiment(EXPERIMENTS_DIR, cfg, prefix='train') as exp: print(f'Experiment started: {exp.experiment_id}') preprocess_exp = Experiment.load(EXPERIMENTS_DIR, exp.config.preprocess_exp_id) dataset_train, dataset_val, dataset_test, vocab, style_vocab, W_emb = load_dataset(preprocess_exp) data_loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=exp.config.batch_size, shuffle=True) data_loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=exp.config.batch_size, shuffle=False) print(f'Data loader: {len(data_loader_train)}, {len(data_loader_val)}') model = create_model(exp.config, vocab, style_vocab, dataset_train.max_len, W_emb) update_function_train, update_function_eval = create_update_function(exp.config, model) trainer = Engine(update_function_train) evaluator = Engine(update_function_eval) metrics = {'loss': LossAggregatorMetric(), } for metric_name, metric in metrics.items(): metric.attach(evaluator, metric_name) best_loss = np.inf @trainer.on(Events.ITERATION_COMPLETED) def log_training_iter(engine): losses_train = engine.state.output log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer, True) @trainer.on(Events.EPOCH_COMPLETED) def log_training_results(engine): nonlocal best_loss # evaluator.run(data_loader_train) # losses_train = evaluator.state.metrics['loss'] evaluator.run(data_loader_val) losses_val = evaluator.state.metrics['loss'] # log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer) log_progress(trainer.state.epoch, trainer.state.iteration, losses_val, 'val', tensorboard_writer) if losses_val[exp.config.best_loss] < best_loss: best_loss = losses_val[exp.config.best_loss] save_weights(model, exp.experiment_dir.joinpath('best.th')) tensorboard_dir = exp.experiment_dir.joinpath('log') tensorboard_writer = SummaryWriter(str(tensorboard_dir)) trainer.run(data_loader_train, max_epochs=exp.config.num_epochs) print(f'Experiment finished: {exp.experiment_id}')
def f_nn(params): """Find the best parameters defined in the space.""" print('Params testing: ', params) # Get dataset dataset_path = os.path.join( os.curdir, 'model/%s/training_data.dat' % params['codebook']) data, label = load_dataset(dataset_path) X, X_val, y, y_val = train_test_split(data, label, test_size=0.3, random_state=0) X, X_val, y, y_val = np.array(X), np.array(X_val), np.array(y), np.array( y_val) model = Sequential() model.add( Dense(int(params['units1']), input_dim=X.shape[1], activation=params['activation'])) model.add(Dropout(params['dropout1'])) model.add(Dense(int(params['units2']), activation=params['activation'])) model.add(Dropout(params['dropout2'])) if params['choice']['layers'] == 'three': model.add( Dense(int(params['choice']['units3']), activation=params['activation'])) model.add(Dropout(params['choice']['dropout3'])) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer=params['optimizer']) model.fit(X, y, nb_epoch=int(params['nb_epochs']), batch_size=int(params['batch_size']), verbose=0) pred_auc = model.predict_proba(X_val, batch_size=128, verbose=0) acc = roc_auc_score(y_val, pred_auc) print('AUC:', acc) sys.stdout.flush() return {'loss': -acc, 'status': STATUS_OK}
#!/usr/bin/env python3 import sys import heapq from preprocess import load_dataset if __name__ == '__main__': n = 3 pfile = open(sys.argv[2]) print("Id,Emoticon") for rid, _ in load_dataset(open(sys.argv[1])): pred = [ str(x[0]) for x in sorted( ((i,float(next(pfile))) for i in range(1, 41)) ,key=lambda x:x[1]) ] print( "{rid},{prediction}".format( rid=rid, prediction=" ".join(reversed(pred)) ))
import json import argparse parser = argparse.ArgumentParser(description='DMN+ Trainer') parser.add_argument('--settings_file', type=str, help='path to a json with settings') settings = parser.parse_args() settings = json.loads(open(settings['--settings_file'], 'r')) print("----- Loading Dataset ----") max_len, trainset, testset = load_dataset( emb_location=settings["path_to_embeddings"], babi_location=settings["path_to_train_task"], babi_test_location=settings["path_to_test_task"], emb_dim=settings["embeddings_size"]) input_shape = trainset[0][0].shape question_shape = trainset[1][0].shape num_classes = len(trainset[2][0]) print("----- Dataset Loaded. Compiling Model -----") dmn_net = DynamicMemoryNetwork(save_folder=model_folder) dmn_net.build_inference_graph(input_shape=input_shape, question_shape=question_shape, num_classes=num_classes, units=settings["hidden_units"], batch_size=settings["batch_size"], memory_steps=settings["memory_steps"],
def main(): tf.enable_eager_execution() parser = argparse.ArgumentParser(description='Translator tester') parser.add_argument("in_seq", type=str, help="sequence to translate") parser.add_argument("-p", "--plot_atention", dest="plot_attention", help="plot attention grid", action="store_true") args = parser.parse_args() model_info = json.load(open('model_info.json', 'r', encoding='UTF-8')) encoder = Encoder(model_info['VIS'], model_info['ED'], model_info['UNITS'], model_info['BZ'], model_info['MAX_SEQ_LEN']) decoder = Decoder(model_info['VTS'], model_info['ED'], model_info['UNITS'], model_info['BZ'], model_info['MAX_SEQ_LEN']) input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset( model_info['DATASET'], model_info['DSS']) evaluate("ve", encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ) encoder.load_weights('encoder') decoder.load_weights('decoder') translate(args.in_seq, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ, args.plot_attention)
experiment.log_parameters(hyperparams) model = GPT24QUAC().to(device) tokenizer = GPT2Tokenizer.from_pretrained('gpt2') tokenizer.add_special_tokens({ "sep_token": "<SEP>", "bos_token": "<BOS>", "eos_token": "<EOS>", "pad_token": "<PAD>" }) tokenizer.add_tokens("CANNOTANSWER") model.resize_token_embeddings(len(tokenizer)) train_loader, test_loader = load_dataset( [args.train_file, args.test_file], tokenizer, batch_size=hyperparams["batch_size"], max_seq_len=hyperparams['max_seq_len'], window_stride=hyperparams['window_stride']) optimizer = optim.Adam(model.parameters(), lr=hyperparams['lr']) if args.load: model.load_state_dict(torch.load('./model.pt')) if args.train: train(model, train_loader, optimizer, experiment, hyperparams) if args.test: test(model, test_loader, tokenizer, experiment, hyperparams) if args.save: torch.save(model.state_dict(), './model.pt')
import json from preprocess import load_dataset, preprocess_data from keras.models import Sequential from keras.layers import Dense, Activation, Conv1D, Dropout, Flatten, AveragePooling1D from keras.layers.embeddings import Embedding # Getting Preprocessed Data # word dictionary length max_words = 10000 # loading data from twitter sentiment analysis dataset print('Loading Data...') X, Y = load_dataset('data/dataset.csv') # preprocessing print('Preprocessing Data...') X_train, Y_train = preprocess_data(X, Y, max_words) # Model Architecture embedding_size = 10 input_dim = max_words + 1 input_length = X_train.shape[1] model = Sequential() model.add( Embedding(input_dim=input_dim, output_dim=embedding_size, input_length=input_length,
import tensorflow as tf import decoder import attention import encoder import preprocess # 加载、预处理数据 input_tensor, target_tensor, inp_lang, targ_lang = preprocess.load_dataset("./cmn.txt", 30000) # 公共参数定义 BUFFER_SIZE = len(input_tensor) BATCH_SIZE = 32 steps_per_epoch = len(input_tensor)//BATCH_SIZE embedding_dim = 256 # 词向量维度 units = 512 vocab_inp_size = len(inp_lang.word_index)+1 vocab_tar_size = len(targ_lang.word_index)+1 # 数据集 dataset = tf.data.Dataset.from_tensor_slices((input_tensor, target_tensor)).shuffle(BUFFER_SIZE) dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) # 输出dataset样例 example_input_batch, example_target_batch = next(iter(dataset)) print(example_input_batch.shape, example_target_batch.shape) # 定义encoder encoder = encoder.Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE) sample_hidden = encoder.initialize_hidden_state() sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)
# x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size) x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1) # pass concatenated vector to GRU output, state = self.gru(x) # output shape == (batch_size * 1, hidden_size) output = tf.reshape(output, (-1, output.shape[2])) # output shape == (batch_size, vocab) x = self.fc(output) return x, state, attention_weights input_tensor, output_tensor, input_line, output_line = load_dataset(FILE_PATH) # Calculate max length of target tensors max_length_output, max_length_input = max_length(output_tensor), max_length( input_tensor) # Create trainining/test set with 80-20 split input_tensor_train, input_tensor_test, output_tensor_train, output_tensor_test = train_test_split( input_tensor, output_tensor, test_size=0.2) # Create tf.data Dataset. Start by creating hyperparameters. BUFFER_SIZE = len(input_tensor_train) BATCH_SIZE = 64 STEPS_PER_EPOCH = len(input_tensor_train) // BATCH_SIZE EMBEDDING_DIM = 256 UNITS = 1024
path_to_zip = tf.keras.utils.get_file( 'spa-eng.zip', origin= 'http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip', extract=True) path_to_file = os.path.dirname(path_to_zip) + "/spa-eng/spa.txt" #Make sure that you have the preprocees.py file which contain some helper functions #Download the file from https://github.com/Mahyar-Ali/Neural-Machine-Translation/blob/master/preprocess.py import preprocess #We will be using the first 50000 examples from the dataset num_examples = 50000 #Using the preprocess module to preprocess the data input_tensor, target_tensor, inp_lang, targ_lang = preprocess.load_dataset( path_to_file, num_examples) max_length_targ, max_length_inp = target_tensor.shape[1], input_tensor.shape[1] # Creating training and validation sets using an 80-20 split input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split( input_tensor, target_tensor, test_size=0.2) # Show length print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)) #This function is to just display how the data is encoded def convert(lang, tensor): for t in tensor: if t != 0:
from keras.models import Model from keras.models import Sequential from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D, BatchNormalization from keras.optimizers import SGD, RMSprop, Adam, Adagrad, Adadelta, RMSprop print("-------------------Libraries Loaded Successfully!!!-------------------------------------------") except: print("Library not Found ! ") #----------------------------------Loading the train data------------------------------------------------------------------------ #Informarion PATH = "D:/Major/Models/lwdct/segmented_frameset_resized/train/" IMAGE_SIZE = 128 # Loading the dataset dat,Y=pp.load_dataset(PATH , IMAGE_SIZE) #Converting labels into one hot labels=np.array(Y) num_classes=6 Y = np_utils.to_categorical(labels, num_classes)#onehot class Labels # Class Labels cl=["HandShaking","Hugging","Kicking","Pointing","Punching","PUSHING"] #----------------Split data into train test-------------------------------------------------------------- epochs = 20 batch_size = int(.25 * 480) if batch_size > 128:
tb_cb = keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=True) ### end #figure model's summary #model.summary() #plot model's overview plot_model(model, show_shapes=True, to_file='model.png') early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=1, mode='auto') cbks = [tb_cb, early_stopping] #cbks = [tb_cb] #training process model.fit_generator(generator=load_dataset(train_list, training_dataset_dir, training_label_png_dir, (input_shape[0], input_shape[1]), NUM_OF_CLASSES, num_batchsizes), steps_per_epoch = num_steps_per_epoch, nb_epoch=num_epoches, validation_data=load_dataset(validation_list, validation_dataset_dir, validation_label_png_dir, (input_shape[0], input_shape[1]), NUM_OF_CLASSES, num_batchsizes), validation_steps=num_val_steps, verbose=2, callbacks=cbks ) print("complete training!") #save weights model.save_weights(weights_name, overwrite=True) print("complete saving the weights")
def train(previous_model=None, epochs=50, initial_epoch=0): """Train the best network model found by hyperparameter optimization. :param previous_model: Path to the initial model weights to be trained. :type previous_model: str :param epochs: The number of epoch for training. :type epochs: int :param initial_epoch: The initial epoch number. :type initial_epoch: int """ # Load training dataset dataset_path = os.path.join(os.curdir, 'model/sift_500/training_data.dat') if not os.path.exists(dataset_path): print 'Dataset: %s under model dir does not exist.' % dataset_path data, label = load_dataset(dataset_path) batch_size = 122 n_feature = data[0].shape[0] # The best network model found by hyperparameter optimization model = Sequential() # 1st hidden layer model.add(Dense(900, input_dim=n_feature, activation='relu')) model.add(Dropout(0.7499998682813903)) # 2nd hidden layer model.add(Dense(609, activation='relu')) model.add(Dropout(0.3780638891568126)) # Output layer model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy']) if previous_model: model.load_weights(previous_model) # Checkpoint filepath = ( 'model/final_model_500/weights-improvement-{epoch:02d}-{val_acc:.2f}.hdf5' ) checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] # Train model.fit(np.array(data), np.array(label), validation_split=0.3, batch_size=batch_size, epochs=epochs, initial_epoch=initial_epoch, callbacks=callbacks_list, shuffle=True, verbose=True) model.save_weights('model/final_model_500/weights-last.hdf5')
# Defining and parsing the command-line arguments parser = argparse.ArgumentParser( description='Preprocessing for training component') parser.add_argument('--output-dataset-path', type=str, help='Path to the preprocessed training dataset') args = parser.parse_args() # Creating the directory where the output file will be created (the directory may or may not exist). Path(args.output_dataset_path).parent.mkdir(parents=True, exist_ok=True) df = preprocess.load_dataset(zipfile_path="/usr/src/app/data/CSV-01-12.zip", metadata_path="/usr/src/app/data/metadata.json", random_state=1, csvs=[ 'UDPLag.csv', 'Syn.csv', 'DrDoS_UDP.csv', 'DrDoS_NetBIOS.csv', 'DrDoS_MSSQL.csv', 'DrDoS_LDAP.csv' ], chunksize=8**6) with open(args.output_dataset_path, 'w') as dataset_file: df.to_csv(dataset_file, index=False) end = time.time() elapsed_time = (end - start) logging.debug('Preprocessing time: {t}'.format(t=datetime.timedelta( seconds=elapsed_time))) logging.debug('END: {t}'.format(t=datetime.datetime.now()))
start = time.time() logging.debug('START: {t}'.format(t=datetime.datetime.now())) # Defining and parsing the command-line arguments parser = argparse.ArgumentParser( description='Preprocessing for testing component') parser.add_argument('--output-dataset-path', type=str, help='Path to the preprocessed testing dataset') args = parser.parse_args() # Creating the directory where the output file will be created (the directory may or may not exist). Path(args.output_dataset_path).parent.mkdir(parents=True, exist_ok=True) df = preprocess.load_dataset(zipfile_path="/usr/src/app/data/CSV-03-11.zip", metadata_path="/usr/src/app/data/metadata.json", csvs=[ 'Syn.csv', 'UDPLag.csv', 'UDP.csv', 'LDAP.csv', 'MSSQL.csv', 'NetBIOS.csv' ], chunksize=8**6) with open(args.output_dataset_path, 'w') as dataset_file: df.to_csv(dataset_file, index=False) end = time.time() elapsed_time = (end - start) logging.debug('Preprocessing time: {t}'.format(t=datetime.timedelta( seconds=elapsed_time))) logging.debug('END: {t}'.format(t=datetime.datetime.now()))
def train(model_name, batch_size, steps_per_epoch, epochs, validation_steps, model_file=None, save_path=None): """Train Train an implemented model. # Arguments model_name: String. The name of an implemented model. batch_size: Integer. Number of patches in each training and validation batch. steps_per_epoch: Integer. Number of training batches in each epoch. epochs: Integer. Number of epochs. validation_steps: Integer. Number of batches for validation after each epoch. model_file: String. Model file as h5 format. If not specified, a new model will be created. save_path: String. Path to save trained model file. If not specified, will save in default path. """ print("- Loading configuration...") if model_name in models_default_params: default_params = models_default_params[model_name] else: print("Error: the model '{}' has not been implemented".format(model_name)) return custom_objects = default_params['custom_objects'] patch_size = default_params['patch_size'] if save_path is None: save_path = default_params['default_path'] if os.path.isfile(save_path): print("Warning: {} is an existing file and will be overwritten.".format(save_path)) print("- Configuration loaded.") print("- Loading datasets...") train_gen = preprocess.load_dataset(batch_size, x_directory = "datasets/Potsdam/Training/RGB/", y_directory = "datasets/Potsdam/Training/Labels/", patch_size = patch_size) val_gen = preprocess.load_dataset(batch_size, x_directory = "datasets/Potsdam/Validation/RGB/", y_directory = "datasets/Potsdam/Validation/Labels/", patch_size = patch_size) print("- Data loaded.") print("- Initialising model...") if(model_file is not None): # Further train existing model model = keras.models.load_model(model_file, custom_objects=custom_objects) else: # Create new model if model_name == 'fcn': model = fcn.make_fcn_resnet((patch_size, patch_size, channels), nb_labels, use_pretraining=False, freeze_base=False) elif model_name == 'pspnet': model = pspnet.build_pspnet(nb_classes=nb_labels, resnet_layers=50, input_shape=patch_size) elif model_name == 'mobilenetv2': model = mobilenetv2.MobileNetv2((patch_size, patch_size, channels), nb_labels) model.compile( optimizer = optimizers.Adam(lr = 0.00001), loss = losses.categorical_crossentropy, metrics = [metrics.categorical_accuracy]) model.summary() print("- Model initialised.") tensorboard = callbacks.TensorBoard(log_dir='./logs') csv_logger = callbacks.CSVLogger('logs/training.csv') checkpoint = callbacks.ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True, save_best_only=True) print("- Starting training.") model.fit_generator( generator=train_gen, steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=val_gen, validation_steps=validation_steps, # callbacks=[checkpoint, csv_logger] ) print("- Training complete.") model.save(save_path) print("- Model saved to {}".format(save_path))
def main() -> NoReturn: """TensorFlow版transformer执行器入口 """ parser = ArgumentParser(description="informer", ) parser.add_argument("--pre_epochs", default=20, type=int, required=False, help="训练步数") parser.add_argument("--epochs", default=10, type=int, required=False, help="训练步数") parser.add_argument("--enc_num_layers", default=3, type=int, required=False, help="encoder和decoder的内部层数") parser.add_argument("--dec_num_layers", default=2, type=int, required=False, help="encoder和decoder的内部层数") parser.add_argument("--num_heads", default=8, type=int, required=False, help="头注意力数量") parser.add_argument("--units", default=1024, type=int, required=False, help="隐藏层单元数") parser.add_argument("--dropout", default=0.1, type=float, required=False, help="dropout") parser.add_argument("--embedding_dim", default=512, type=int, required=False, help="嵌入层维度大小") parser.add_argument("--batch_size", default=1, type=int, required=False, help="batch大小") parser.add_argument("--buffer_size", default=10000, type=int, required=False, help="Dataset加载缓冲大小") parser.add_argument("--checkpoint_save_size", default=20, type=int, required=False, help="单轮训练中检查点保存数量") parser.add_argument("--checkpoint_save_freq", default=1, type=int, required=False, help="检查点保存频率") parser.add_argument("--checkpoint_dir", default="checkpoint/", type=str, required=False, help="") parser.add_argument("--save_dir", default="user_data/train/", type=str, required=False, help="") parser.add_argument("--result_save_path", default="result/", type=str, required=False, help="") parser.add_argument("--save_pairs", default="user_data/pairs.txt", type=str, required=False, help="") parser.add_argument("--save_soda_pairs", default="user_data/soda_pairs.txt", type=str, required=False, help="") parser.add_argument("--save_cmip_pairs", default="user_data/cmip_pairs.txt", type=str, required=False, help="") parser.add_argument( "--soda_train_data_path", default="tcdata/enso_round1_train_20210201/SODA_train.nc", type=str, required=False, help="") parser.add_argument( "--soda_label_data_path", default="tcdata/enso_round1_train_20210201/SODA_label.nc", type=str, required=False, help="") parser.add_argument( "--cmip_train_data_path", default="tcdata/enso_round1_train_20210201/CMIP_train.nc", type=str, required=False, help="") parser.add_argument( "--cmip_label_data_path", default="tcdata/enso_round1_train_20210201/CMIP_label.nc", type=str, required=False, help="") parser.add_argument("--test_data_path", default="tcdata/enso_final_test_data_B/", type=str, required=False, help="") options = parser.parse_args() print("正在处理soda语料") preprocess_soda(train_data_path=options.soda_train_data_path, label_data_path=options.soda_label_data_path, save_pairs=options.save_soda_pairs, save_dir=options.save_dir) print("正在处理cmip语料") preprocess_cmip(train_data_path=options.cmip_train_data_path, label_data_path=options.cmip_label_data_path, save_pairs=options.save_cmip_pairs, save_dir=options.save_dir) print("加载模型中") informer_model = informer(embedding_dim=options.embedding_dim, enc_num_layers=options.enc_num_layers, dec_num_layers=options.dec_num_layers, batch_size=options.batch_size, num_heads=options.num_heads, dropout=options.dropout) checkpoint_manager = load_checkpoint( checkpoint_dir=options.checkpoint_dir, checkpoint_save_size=options.checkpoint_save_size, model=informer_model) print("正在cmip预训练") cmip_train_dataset, _ = load_dataset(pairs_path=options.save_cmip_pairs, batch_size=options.batch_size, buffer_size=options.buffer_size, data_type="cmip") _ = train(model=informer_model, checkpoint=checkpoint_manager, batch_size=options.batch_size, epochs=options.pre_epochs, train_dataset=cmip_train_dataset, valid_dataset=None, checkpoint_save_freq=options.checkpoint_save_freq) print("正在进行soda微调训练") soda_train_dataset, soda_valid_dataset = load_dataset( pairs_path=options.save_cmip_pairs, batch_size=options.batch_size, buffer_size=options.buffer_size, data_type="soda") _ = train(model=informer_model, checkpoint=checkpoint_manager, batch_size=options.batch_size, epochs=options.epochs, train_dataset=soda_train_dataset, valid_dataset=soda_valid_dataset, checkpoint_save_freq=options.checkpoint_save_freq) print("正在预测中") inference(model=informer_model, result_save_path=options.result_save_path, test_data_path=options.test_data_path)
torch.manual_seed(RANDOM_STATE) np.random.seed(RANDOM_STATE) device = torch.device("cpu") annotations = pd.read_csv("./data/train.csv") data_raw_path = "./data/train" data_processed_path = "./processed_data/" # We load our dataset once with preprocessing then # load directly the training and validation tensors for faster training get_data = False if get_data: X, Y = load_dataset(data_raw_path, annotations) X_train, X_valid, Y_train, Y_valid = train_test_split( X, Y, test_size=0.2, random_state=RANDOM_STATE, stratify=Y) X_train = torch.from_numpy(X_train).float() X_valid = torch.from_numpy(X_valid).float() Y_train = torch.from_numpy(Y_train).float() Y_valid = torch.from_numpy(Y_valid).float() train = torch.utils.data.TensorDataset(X_train, Y_train) valid = torch.utils.data.TensorDataset(X_valid, Y_valid) # save training and validation datasets torch.save(train, data_processed_path + "train.pth") torch.save(valid, data_processed_path + "valid.pth")
# -*- coding: utf-8 -*- """ Code for paper: Ghods, Ramina and Lan, Andrew S and Goldstein, Tom and Studer, Christoph, ``MSE-optimal neural network initialization via layer fusion", 2020 54th Annual Conference on Information Sciences and Systems (CISS) (c) 2020 raminaghods ([email protected]) Preprocess the speech data -- only need to run this once to get the data_arrays code for data processing from: P. Warden, “Speech commands: A dataset for limited-vocabulary speech recognition,” arXiv preprint: 1804.03209, Apr. 2018. """ from preprocess import load_dataset, save_data_to_array seq_len = 128 # Number of steps, max of 128 n_channels = 11 # number of input channels maximum 11 DATA_PATH = "ENTER DATA PATH" load_dataset(path=DATA_PATH, seq_len=seq_len) # Save data to array file first save_data_to_array(path=DATA_PATH, max_len=n_channels, seq_len=seq_len)
torch.load(args.classifier_path + "/baseline/emb.pt")) vocab_classifier1 = pkl.load(open(args.classifier_path + "/vocab.pkl", 'rb')) mlp_classifier = MLPClassifier(args.z_size * 2, 3, layers=args.surrogate_layers) if not args.train_mode: mlp_classifier.load_state_dict( torch.load(args.save_path + '/surrogate{0}.pt'.format(args.surrogate_layers))) #---------------------------------------------- exp_id = 'train.3ir9y_e3' exp = Experiment.load(EXPERIMENTS_DIR, exp_id) preprocess_exp = Experiment.load(EXPERIMENTS_DIR, exp.config.preprocess_exp_id) dataset_train, dataset_val, dataset_test, vocab, style_vocab, W_emb = load_dataset( preprocess_exp) dataset_reader = create_dataset_reader(preprocess_exp.config) model = create_model(exp.config, vocab, style_vocab, dataset_train.max_len, W_emb) load_weights(model, exp.experiment_dir.joinpath('best.th')) #--------------------------------------------- print(classifier1) print(autoencoder) print(inverter) print(mlp_classifier) optimizer = optim.Adam(mlp_classifier.parameters(), lr=args.lr, betas=(args.beta1, 0.999))