def get_dataset(dataset_path, batch_size, strategy=None):
# Load dataset
dataset = preprocess.load_dataset(dataset_path)
# Get max length
max_mel_length = 0
max_token_length = 0
for b, (mel, token) in enumerate(dataset):
max_mel_length = max(max_mel_length, mel.shape[0])
max_token_length = max(max_token_length, token.shape[0])
print("Max mel length :", max_mel_length)
print("Max token length :", max_token_length)
num_mels = mel.shape[1]
# Pad for batching
dataset = dataset.padded_batch(
batch_size,
padded_shapes=(
[max_mel_length, num_mels], # mel_specs
[max_token_length]) # labels
)
# Prefetch dataset (cuda streaming)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
if strategy is not None:
dataset = strategy.experimental_distribute_dataset(dataset)
return dataset
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
dataset = preprocessor.get_dataset(
preprocessor.load_dataset(FLAGS.dataset_path),
preprocessor.load_json(FLAGS.split_path))
preprocessor.write_dataset(dataset, FLAGS.save_path)
token_vocab = preprocessor.get_token_vocab(FLAGS.save_path)
preprocessor.write_token_vocab(token_vocab, FLAGS.save_path)
def main(cfg):
with Experiment(EXPERIMENTS_DIR, cfg, prefix='train') as exp:
print(f'Experiment started: {exp.experiment_id}')
preprocess_exp = Experiment.load(EXPERIMENTS_DIR, exp.config.preprocess_exp_id)
dataset_train, dataset_val, dataset_test, vocab, style_vocab, W_emb = load_dataset(preprocess_exp)
data_loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=exp.config.batch_size, shuffle=True)
data_loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=exp.config.batch_size, shuffle=False)
print(f'Data loader: {len(data_loader_train)}, {len(data_loader_val)}')
model = create_model(exp.config, vocab, style_vocab, dataset_train.max_len, W_emb)
update_function_train, update_function_eval = create_update_function(exp.config, model)
trainer = Engine(update_function_train)
evaluator = Engine(update_function_eval)
metrics = {'loss': LossAggregatorMetric(), }
for metric_name, metric in metrics.items():
metric.attach(evaluator, metric_name)
best_loss = np.inf
@trainer.on(Events.ITERATION_COMPLETED)
def log_training_iter(engine):
losses_train = engine.state.output
log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer, True)
@trainer.on(Events.EPOCH_COMPLETED)
def log_training_results(engine):
nonlocal best_loss
# evaluator.run(data_loader_train)
# losses_train = evaluator.state.metrics['loss']
evaluator.run(data_loader_val)
losses_val = evaluator.state.metrics['loss']
# log_progress(trainer.state.epoch, trainer.state.iteration, losses_train, 'train', tensorboard_writer)
log_progress(trainer.state.epoch, trainer.state.iteration, losses_val, 'val', tensorboard_writer)
if losses_val[exp.config.best_loss] < best_loss:
best_loss = losses_val[exp.config.best_loss]
save_weights(model, exp.experiment_dir.joinpath('best.th'))
tensorboard_dir = exp.experiment_dir.joinpath('log')
tensorboard_writer = SummaryWriter(str(tensorboard_dir))
trainer.run(data_loader_train, max_epochs=exp.config.num_epochs)
print(f'Experiment finished: {exp.experiment_id}')
def f_nn(params):
"""Find the best parameters defined in the space."""
print('Params testing: ', params)
# Get dataset
dataset_path = os.path.join(
os.curdir, 'model/%s/training_data.dat' % params['codebook'])
data, label = load_dataset(dataset_path)
X, X_val, y, y_val = train_test_split(data,
label,
test_size=0.3,
random_state=0)
X, X_val, y, y_val = np.array(X), np.array(X_val), np.array(y), np.array(
y_val)
model = Sequential()
model.add(
Dense(int(params['units1']),
input_dim=X.shape[1],
activation=params['activation']))
model.add(Dropout(params['dropout1']))
model.add(Dense(int(params['units2']), activation=params['activation']))
model.add(Dropout(params['dropout2']))
if params['choice']['layers'] == 'three':
model.add(
Dense(int(params['choice']['units3']),
activation=params['activation']))
model.add(Dropout(params['choice']['dropout3']))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=params['optimizer'])
model.fit(X,
y,
nb_epoch=int(params['nb_epochs']),
batch_size=int(params['batch_size']),
verbose=0)
pred_auc = model.predict_proba(X_val, batch_size=128, verbose=0)
acc = roc_auc_score(y_val, pred_auc)
print('AUC:', acc)
sys.stdout.flush()
return {'loss': -acc, 'status': STATUS_OK}
#!/usr/bin/env python3
import sys
import heapq
from preprocess import load_dataset
if __name__ == '__main__':
n = 3
pfile = open(sys.argv[2])
print("Id,Emoticon")
for rid, _ in load_dataset(open(sys.argv[1])):
pred = [
str(x[0]) for x in
sorted(
((i,float(next(pfile))) for i in range(1, 41))
,key=lambda x:x[1])
]
print( "{rid},{prediction}".format(
rid=rid,
prediction=" ".join(reversed(pred))
))
import json
import argparse
parser = argparse.ArgumentParser(description='DMN+ Trainer')
parser.add_argument('--settings_file',
type=str,
help='path to a json with settings')
settings = parser.parse_args()
settings = json.loads(open(settings['--settings_file'], 'r'))
print("----- Loading Dataset ----")
max_len, trainset, testset = load_dataset(
emb_location=settings["path_to_embeddings"],
babi_location=settings["path_to_train_task"],
babi_test_location=settings["path_to_test_task"],
emb_dim=settings["embeddings_size"])
input_shape = trainset[0][0].shape
question_shape = trainset[1][0].shape
num_classes = len(trainset[2][0])
print("----- Dataset Loaded. Compiling Model -----")
dmn_net = DynamicMemoryNetwork(save_folder=model_folder)
dmn_net.build_inference_graph(input_shape=input_shape,
question_shape=question_shape,
num_classes=num_classes,
units=settings["hidden_units"],
batch_size=settings["batch_size"],
memory_steps=settings["memory_steps"],
def main():
tf.enable_eager_execution()
parser = argparse.ArgumentParser(description='Translator tester')
parser.add_argument("in_seq", type=str, help="sequence to translate")
parser.add_argument("-p",
"--plot_atention",
dest="plot_attention",
help="plot attention grid",
action="store_true")
args = parser.parse_args()
model_info = json.load(open('model_info.json', 'r', encoding='UTF-8'))
encoder = Encoder(model_info['VIS'], model_info['ED'], model_info['UNITS'],
model_info['BZ'], model_info['MAX_SEQ_LEN'])
decoder = Decoder(model_info['VTS'], model_info['ED'], model_info['UNITS'],
model_info['BZ'], model_info['MAX_SEQ_LEN'])
input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset(
model_info['DATASET'], model_info['DSS'])
evaluate("ve", encoder, decoder, inp_lang, targ_lang, max_length_inp,
max_length_targ)
encoder.load_weights('encoder')
decoder.load_weights('decoder')
translate(args.in_seq, encoder, decoder, inp_lang, targ_lang,
max_length_inp, max_length_targ, args.plot_attention)
experiment.log_parameters(hyperparams)
model = GPT24QUAC().to(device)
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
tokenizer.add_special_tokens({
"sep_token": "<SEP>",
"bos_token": "<BOS>",
"eos_token": "<EOS>",
"pad_token": "<PAD>"
})
tokenizer.add_tokens("CANNOTANSWER")
model.resize_token_embeddings(len(tokenizer))
train_loader, test_loader = load_dataset(
[args.train_file, args.test_file],
tokenizer,
batch_size=hyperparams["batch_size"],
max_seq_len=hyperparams['max_seq_len'],
window_stride=hyperparams['window_stride'])
optimizer = optim.Adam(model.parameters(), lr=hyperparams['lr'])
if args.load:
model.load_state_dict(torch.load('./model.pt'))
if args.train:
train(model, train_loader, optimizer, experiment, hyperparams)
if args.test:
test(model, test_loader, tokenizer, experiment, hyperparams)
if args.save:
torch.save(model.state_dict(), './model.pt')
import json
from preprocess import load_dataset, preprocess_data
from keras.models import Sequential
from keras.layers import Dense, Activation, Conv1D, Dropout, Flatten, AveragePooling1D
from keras.layers.embeddings import Embedding
# Getting Preprocessed Data
# word dictionary length
max_words = 10000
# loading data from twitter sentiment analysis dataset
print('Loading Data...')
X, Y = load_dataset('data/dataset.csv')
# preprocessing
print('Preprocessing Data...')
X_train, Y_train = preprocess_data(X, Y, max_words)
# Model Architecture
embedding_size = 10
input_dim = max_words + 1
input_length = X_train.shape[1]
model = Sequential()
model.add(
Embedding(input_dim=input_dim,
output_dim=embedding_size,
input_length=input_length,
import tensorflow as tf
import decoder
import attention
import encoder
import preprocess
# 加载、预处理数据
input_tensor, target_tensor, inp_lang, targ_lang = preprocess.load_dataset("./cmn.txt", 30000)
# 公共参数定义
BUFFER_SIZE = len(input_tensor)
BATCH_SIZE = 32
steps_per_epoch = len(input_tensor)//BATCH_SIZE
embedding_dim = 256 # 词向量维度
units = 512
vocab_inp_size = len(inp_lang.word_index)+1
vocab_tar_size = len(targ_lang.word_index)+1
# 数据集
dataset = tf.data.Dataset.from_tensor_slices((input_tensor, target_tensor)).shuffle(BUFFER_SIZE)
dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)
# 输出dataset样例
example_input_batch, example_target_batch = next(iter(dataset))
print(example_input_batch.shape, example_target_batch.shape)
# 定义encoder
encoder = encoder.Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)
sample_hidden = encoder.initialize_hidden_state()
sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)
# x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)
x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)
# pass concatenated vector to GRU
output, state = self.gru(x)
# output shape == (batch_size * 1, hidden_size)
output = tf.reshape(output, (-1, output.shape[2]))
# output shape == (batch_size, vocab)
x = self.fc(output)
return x, state, attention_weights
input_tensor, output_tensor, input_line, output_line = load_dataset(FILE_PATH)
# Calculate max length of target tensors
max_length_output, max_length_input = max_length(output_tensor), max_length(
input_tensor)
# Create trainining/test set with 80-20 split
input_tensor_train, input_tensor_test, output_tensor_train, output_tensor_test = train_test_split(
input_tensor, output_tensor, test_size=0.2)
# Create tf.data Dataset. Start by creating hyperparameters.
BUFFER_SIZE = len(input_tensor_train)
BATCH_SIZE = 64
STEPS_PER_EPOCH = len(input_tensor_train) // BATCH_SIZE
EMBEDDING_DIM = 256
UNITS = 1024
path_to_zip = tf.keras.utils.get_file(
'spa-eng.zip',
origin=
'http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip',
extract=True)
path_to_file = os.path.dirname(path_to_zip) + "/spa-eng/spa.txt"
#Make sure that you have the preprocees.py file which contain some helper functions
#Download the file from https://github.com/Mahyar-Ali/Neural-Machine-Translation/blob/master/preprocess.py
import preprocess
#We will be using the first 50000 examples from the dataset
num_examples = 50000
#Using the preprocess module to preprocess the data
input_tensor, target_tensor, inp_lang, targ_lang = preprocess.load_dataset(
path_to_file, num_examples)
max_length_targ, max_length_inp = target_tensor.shape[1], input_tensor.shape[1]
# Creating training and validation sets using an 80-20 split
input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(
input_tensor, target_tensor, test_size=0.2)
# Show length
print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val),
len(target_tensor_val))
#This function is to just display how the data is encoded
def convert(lang, tensor):
for t in tensor:
if t != 0:
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from keras.optimizers import SGD, RMSprop, Adam, Adagrad, Adadelta, RMSprop
print("-------------------Libraries Loaded Successfully!!!-------------------------------------------")
except:
print("Library not Found ! ")
#----------------------------------Loading the train data------------------------------------------------------------------------
#Informarion
PATH = "D:/Major/Models/lwdct/segmented_frameset_resized/train/"
IMAGE_SIZE = 128
# Loading the dataset
dat,Y=pp.load_dataset(PATH , IMAGE_SIZE)
#Converting labels into one hot
labels=np.array(Y)
num_classes=6
Y = np_utils.to_categorical(labels, num_classes)#onehot class Labels
# Class Labels
cl=["HandShaking","Hugging","Kicking","Pointing","Punching","PUSHING"]
#----------------Split data into train test--------------------------------------------------------------
epochs = 20
batch_size = int(.25 * 480)
if batch_size > 128:
tb_cb = keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=True, write_images=True)
### end
#figure model's summary
#model.summary()
#plot model's overview
plot_model(model, show_shapes=True, to_file='model.png')
early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=1, mode='auto')
cbks = [tb_cb, early_stopping]
#cbks = [tb_cb]
#training process
model.fit_generator(generator=load_dataset(train_list, training_dataset_dir, training_label_png_dir, (input_shape[0], input_shape[1]), NUM_OF_CLASSES, num_batchsizes),
steps_per_epoch = num_steps_per_epoch,
nb_epoch=num_epoches,
validation_data=load_dataset(validation_list, validation_dataset_dir, validation_label_png_dir, (input_shape[0], input_shape[1]), NUM_OF_CLASSES, num_batchsizes),
validation_steps=num_val_steps,
verbose=2,
callbacks=cbks
)
print("complete training!")
#save weights
model.save_weights(weights_name, overwrite=True)
print("complete saving the weights")
def train(previous_model=None, epochs=50, initial_epoch=0):
"""Train the best network model found by hyperparameter optimization.
:param previous_model: Path to the initial model weights to be trained.
:type previous_model: str
:param epochs: The number of epoch for training.
:type epochs: int
:param initial_epoch: The initial epoch number.
:type initial_epoch: int
"""
# Load training dataset
dataset_path = os.path.join(os.curdir, 'model/sift_500/training_data.dat')
if not os.path.exists(dataset_path):
print 'Dataset: %s under model dir does not exist.' % dataset_path
data, label = load_dataset(dataset_path)
batch_size = 122
n_feature = data[0].shape[0]
# The best network model found by hyperparameter optimization
model = Sequential()
# 1st hidden layer
model.add(Dense(900, input_dim=n_feature, activation='relu'))
model.add(Dropout(0.7499998682813903))
# 2nd hidden layer
model.add(Dense(609, activation='relu'))
model.add(Dropout(0.3780638891568126))
# Output layer
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
if previous_model:
model.load_weights(previous_model)
# Checkpoint
filepath = (
'model/final_model_500/weights-improvement-{epoch:02d}-{val_acc:.2f}.hdf5'
)
checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max')
callbacks_list = [checkpoint]
# Train
model.fit(np.array(data),
np.array(label),
validation_split=0.3,
batch_size=batch_size,
epochs=epochs,
initial_epoch=initial_epoch,
callbacks=callbacks_list,
shuffle=True,
verbose=True)
model.save_weights('model/final_model_500/weights-last.hdf5')
# Defining and parsing the command-line arguments
parser = argparse.ArgumentParser(
description='Preprocessing for training component')
parser.add_argument('--output-dataset-path',
type=str,
help='Path to the preprocessed training dataset')
args = parser.parse_args()
# Creating the directory where the output file will be created (the directory may or may not exist).
Path(args.output_dataset_path).parent.mkdir(parents=True, exist_ok=True)
df = preprocess.load_dataset(zipfile_path="/usr/src/app/data/CSV-01-12.zip",
metadata_path="/usr/src/app/data/metadata.json",
random_state=1,
csvs=[
'UDPLag.csv', 'Syn.csv', 'DrDoS_UDP.csv',
'DrDoS_NetBIOS.csv', 'DrDoS_MSSQL.csv',
'DrDoS_LDAP.csv'
],
chunksize=8**6)
with open(args.output_dataset_path, 'w') as dataset_file:
df.to_csv(dataset_file, index=False)
end = time.time()
elapsed_time = (end - start)
logging.debug('Preprocessing time: {t}'.format(t=datetime.timedelta(
seconds=elapsed_time)))
logging.debug('END: {t}'.format(t=datetime.datetime.now()))
start = time.time()
logging.debug('START: {t}'.format(t=datetime.datetime.now()))
# Defining and parsing the command-line arguments
parser = argparse.ArgumentParser(
description='Preprocessing for testing component')
parser.add_argument('--output-dataset-path',
type=str,
help='Path to the preprocessed testing dataset')
args = parser.parse_args()
# Creating the directory where the output file will be created (the directory may or may not exist).
Path(args.output_dataset_path).parent.mkdir(parents=True, exist_ok=True)
df = preprocess.load_dataset(zipfile_path="/usr/src/app/data/CSV-03-11.zip",
metadata_path="/usr/src/app/data/metadata.json",
csvs=[
'Syn.csv', 'UDPLag.csv', 'UDP.csv',
'LDAP.csv', 'MSSQL.csv', 'NetBIOS.csv'
],
chunksize=8**6)
with open(args.output_dataset_path, 'w') as dataset_file:
df.to_csv(dataset_file, index=False)
end = time.time()
elapsed_time = (end - start)
logging.debug('Preprocessing time: {t}'.format(t=datetime.timedelta(
seconds=elapsed_time)))
logging.debug('END: {t}'.format(t=datetime.datetime.now()))
def train(model_name, batch_size, steps_per_epoch, epochs, validation_steps,
model_file=None, save_path=None):
"""Train
Train an implemented model.
# Arguments
model_name: String. The name of an implemented model.
batch_size: Integer. Number of patches in each training and validation batch.
steps_per_epoch: Integer. Number of training batches in each epoch.
epochs: Integer. Number of epochs.
validation_steps: Integer. Number of batches for validation after
each epoch.
model_file: String. Model file as h5 format. If not specified, a new
model will be created.
save_path: String. Path to save trained model file. If not specified,
will save in default path.
"""
print("- Loading configuration...")
if model_name in models_default_params:
default_params = models_default_params[model_name]
else:
print("Error: the model '{}' has not been implemented".format(model_name))
return
custom_objects = default_params['custom_objects']
patch_size = default_params['patch_size']
if save_path is None:
save_path = default_params['default_path']
if os.path.isfile(save_path):
print("Warning: {} is an existing file and will be overwritten.".format(save_path))
print("- Configuration loaded.")
print("- Loading datasets...")
train_gen = preprocess.load_dataset(batch_size, x_directory = "datasets/Potsdam/Training/RGB/",
y_directory = "datasets/Potsdam/Training/Labels/",
patch_size = patch_size)
val_gen = preprocess.load_dataset(batch_size, x_directory = "datasets/Potsdam/Validation/RGB/",
y_directory = "datasets/Potsdam/Validation/Labels/",
patch_size = patch_size)
print("- Data loaded.")
print("- Initialising model...")
if(model_file is not None): # Further train existing model
model = keras.models.load_model(model_file, custom_objects=custom_objects)
else: # Create new model
if model_name == 'fcn':
model = fcn.make_fcn_resnet((patch_size, patch_size, channels), nb_labels,
use_pretraining=False, freeze_base=False)
elif model_name == 'pspnet':
model = pspnet.build_pspnet(nb_classes=nb_labels, resnet_layers=50,
input_shape=patch_size)
elif model_name == 'mobilenetv2':
model = mobilenetv2.MobileNetv2((patch_size, patch_size, channels), nb_labels)
model.compile(
optimizer = optimizers.Adam(lr = 0.00001),
loss = losses.categorical_crossentropy,
metrics = [metrics.categorical_accuracy])
model.summary()
print("- Model initialised.")
tensorboard = callbacks.TensorBoard(log_dir='./logs')
csv_logger = callbacks.CSVLogger('logs/training.csv')
checkpoint = callbacks.ModelCheckpoint(filepath=checkpoint_path,
save_weights_only=True,
save_best_only=True)
print("- Starting training.")
model.fit_generator(
generator=train_gen,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=val_gen,
validation_steps=validation_steps,
# callbacks=[checkpoint, csv_logger]
)
print("- Training complete.")
model.save(save_path)
print("- Model saved to {}".format(save_path))
def main() -> NoReturn:
"""TensorFlow版transformer执行器入口
"""
parser = ArgumentParser(description="informer", )
parser.add_argument("--pre_epochs",
default=20,
type=int,
required=False,
help="训练步数")
parser.add_argument("--epochs",
default=10,
type=int,
required=False,
help="训练步数")
parser.add_argument("--enc_num_layers",
default=3,
type=int,
required=False,
help="encoder和decoder的内部层数")
parser.add_argument("--dec_num_layers",
default=2,
type=int,
required=False,
help="encoder和decoder的内部层数")
parser.add_argument("--num_heads",
default=8,
type=int,
required=False,
help="头注意力数量")
parser.add_argument("--units",
default=1024,
type=int,
required=False,
help="隐藏层单元数")
parser.add_argument("--dropout",
default=0.1,
type=float,
required=False,
help="dropout")
parser.add_argument("--embedding_dim",
default=512,
type=int,
required=False,
help="嵌入层维度大小")
parser.add_argument("--batch_size",
default=1,
type=int,
required=False,
help="batch大小")
parser.add_argument("--buffer_size",
default=10000,
type=int,
required=False,
help="Dataset加载缓冲大小")
parser.add_argument("--checkpoint_save_size",
default=20,
type=int,
required=False,
help="单轮训练中检查点保存数量")
parser.add_argument("--checkpoint_save_freq",
default=1,
type=int,
required=False,
help="检查点保存频率")
parser.add_argument("--checkpoint_dir",
default="checkpoint/",
type=str,
required=False,
help="")
parser.add_argument("--save_dir",
default="user_data/train/",
type=str,
required=False,
help="")
parser.add_argument("--result_save_path",
default="result/",
type=str,
required=False,
help="")
parser.add_argument("--save_pairs",
default="user_data/pairs.txt",
type=str,
required=False,
help="")
parser.add_argument("--save_soda_pairs",
default="user_data/soda_pairs.txt",
type=str,
required=False,
help="")
parser.add_argument("--save_cmip_pairs",
default="user_data/cmip_pairs.txt",
type=str,
required=False,
help="")
parser.add_argument(
"--soda_train_data_path",
default="tcdata/enso_round1_train_20210201/SODA_train.nc",
type=str,
required=False,
help="")
parser.add_argument(
"--soda_label_data_path",
default="tcdata/enso_round1_train_20210201/SODA_label.nc",
type=str,
required=False,
help="")
parser.add_argument(
"--cmip_train_data_path",
default="tcdata/enso_round1_train_20210201/CMIP_train.nc",
type=str,
required=False,
help="")
parser.add_argument(
"--cmip_label_data_path",
default="tcdata/enso_round1_train_20210201/CMIP_label.nc",
type=str,
required=False,
help="")
parser.add_argument("--test_data_path",
default="tcdata/enso_final_test_data_B/",
type=str,
required=False,
help="")
options = parser.parse_args()
print("正在处理soda语料")
preprocess_soda(train_data_path=options.soda_train_data_path,
label_data_path=options.soda_label_data_path,
save_pairs=options.save_soda_pairs,
save_dir=options.save_dir)
print("正在处理cmip语料")
preprocess_cmip(train_data_path=options.cmip_train_data_path,
label_data_path=options.cmip_label_data_path,
save_pairs=options.save_cmip_pairs,
save_dir=options.save_dir)
print("加载模型中")
informer_model = informer(embedding_dim=options.embedding_dim,
enc_num_layers=options.enc_num_layers,
dec_num_layers=options.dec_num_layers,
batch_size=options.batch_size,
num_heads=options.num_heads,
dropout=options.dropout)
checkpoint_manager = load_checkpoint(
checkpoint_dir=options.checkpoint_dir,
checkpoint_save_size=options.checkpoint_save_size,
model=informer_model)
print("正在cmip预训练")
cmip_train_dataset, _ = load_dataset(pairs_path=options.save_cmip_pairs,
batch_size=options.batch_size,
buffer_size=options.buffer_size,
data_type="cmip")
_ = train(model=informer_model,
checkpoint=checkpoint_manager,
batch_size=options.batch_size,
epochs=options.pre_epochs,
train_dataset=cmip_train_dataset,
valid_dataset=None,
checkpoint_save_freq=options.checkpoint_save_freq)
print("正在进行soda微调训练")
soda_train_dataset, soda_valid_dataset = load_dataset(
pairs_path=options.save_cmip_pairs,
batch_size=options.batch_size,
buffer_size=options.buffer_size,
data_type="soda")
_ = train(model=informer_model,
checkpoint=checkpoint_manager,
batch_size=options.batch_size,
epochs=options.epochs,
train_dataset=soda_train_dataset,
valid_dataset=soda_valid_dataset,
checkpoint_save_freq=options.checkpoint_save_freq)
print("正在预测中")
inference(model=informer_model,
result_save_path=options.result_save_path,
test_data_path=options.test_data_path)
torch.manual_seed(RANDOM_STATE)
np.random.seed(RANDOM_STATE)
device = torch.device("cpu")
annotations = pd.read_csv("./data/train.csv")
data_raw_path = "./data/train"
data_processed_path = "./processed_data/"
# We load our dataset once with preprocessing then
# load directly the training and validation tensors for faster training
get_data = False
if get_data:
X, Y = load_dataset(data_raw_path, annotations)
X_train, X_valid, Y_train, Y_valid = train_test_split(
X, Y, test_size=0.2, random_state=RANDOM_STATE, stratify=Y)
X_train = torch.from_numpy(X_train).float()
X_valid = torch.from_numpy(X_valid).float()
Y_train = torch.from_numpy(Y_train).float()
Y_valid = torch.from_numpy(Y_valid).float()
train = torch.utils.data.TensorDataset(X_train, Y_train)
valid = torch.utils.data.TensorDataset(X_valid, Y_valid)
# save training and validation datasets
torch.save(train, data_processed_path + "train.pth")
torch.save(valid, data_processed_path + "valid.pth")
# -*- coding: utf-8 -*- """ Code for paper: Ghods, Ramina and Lan, Andrew S and Goldstein, Tom and Studer, Christoph, ``MSE-optimal neural network initialization via layer fusion", 2020 54th Annual Conference on Information Sciences and Systems (CISS) (c) 2020 raminaghods ([email protected]) Preprocess the speech data -- only need to run this once to get the data_arrays code for data processing from: P. Warden, “Speech commands: A dataset for limited-vocabulary speech recognition,” arXiv preprint: 1804.03209, Apr. 2018. """ from preprocess import load_dataset, save_data_to_array seq_len = 128 # Number of steps, max of 128 n_channels = 11 # number of input channels maximum 11 DATA_PATH = "ENTER DATA PATH" load_dataset(path=DATA_PATH, seq_len=seq_len) # Save data to array file first save_data_to_array(path=DATA_PATH, max_len=n_channels, seq_len=seq_len)
torch.load(args.classifier_path + "/baseline/emb.pt"))
vocab_classifier1 = pkl.load(open(args.classifier_path + "/vocab.pkl", 'rb'))
mlp_classifier = MLPClassifier(args.z_size * 2,
3,
layers=args.surrogate_layers)
if not args.train_mode:
mlp_classifier.load_state_dict(
torch.load(args.save_path +
'/surrogate{0}.pt'.format(args.surrogate_layers)))
#----------------------------------------------
exp_id = 'train.3ir9y_e3'
exp = Experiment.load(EXPERIMENTS_DIR, exp_id)
preprocess_exp = Experiment.load(EXPERIMENTS_DIR, exp.config.preprocess_exp_id)
dataset_train, dataset_val, dataset_test, vocab, style_vocab, W_emb = load_dataset(
preprocess_exp)
dataset_reader = create_dataset_reader(preprocess_exp.config)
model = create_model(exp.config, vocab, style_vocab, dataset_train.max_len,
W_emb)
load_weights(model, exp.experiment_dir.joinpath('best.th'))
#---------------------------------------------
print(classifier1)
print(autoencoder)
print(inverter)
print(mlp_classifier)
optimizer = optim.Adam(mlp_classifier.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999))