def run(action, root_dir, data_key, model_key, save_path):
run_name = "_".join((model_key, data_key))
config = init_wandb_session(run_name, action)
criterion = CrossEntropyLoss()
model_path = join(save_path, run_name + '.pt')
if action == 'train':
logging.info("Training the model: {} with dataset: {}".format(model_key, data_key))
train_set, val_set = get_dataset(data_key, root_dir, True)
model = NETWORKS[model_key](input_filters=train_set[0][0].shape[0], num_classes=len(train_set.classes))
model.to(run_device)
train_loader = DataLoader(train_set, batch_size=config.batch_size, shuffle=True)
val_loader = DataLoader(val_set, batch_size=config.batch_size, shuffle=False)
optimizer = SGD(model.parameters(), lr=config.learning_rate, momentum=config.momentum)
scheduler = ReduceLROnPlateau(optimizer, 'min', min_lr=config.min_learning_rate, patience=10)
makedirs(save_path, exist_ok=True)
train(model, train_loader, val_loader, criterion, optimizer, scheduler, config.num_epochs, model_path)
else:
logging.info("Testing the model: {} with dataset: {}".format(model_key, data_key))
model = torch.load(model_path)
test_set = get_dataset(data_key, root_dir, False)
test_loader = DataLoader(test_set, batch_size=config.batch_size, shuffle=False)
metrics = predict(model, test_loader, criterion)
log_pred_metrics(metrics)
def run_model(device,
train_loader,
val_loader,
epochs,
learning_rate,
momentum,
dropout,
step_size,
gamma,
l1=0.0,
l2=0.0):
losses = []
accuracies = []
incorrect_samples = []
print('\nCreating model')
model = Net(dropout).to(device) # Create model
model_summary(model) # Display model summary
optimizer = sgd_optimizer(model, learning_rate, l2,
momentum) # Create optimizer
scheduler = lr_scheduler(optimizer, step_size, gamma) # Set LR scheduler
for epoch in range(1, epochs + 1):
print(f'Epoch {epoch}:')
train(model, device, train_loader, optimizer, epoch, l1)
scheduler.step()
val(model, device, val_loader, losses, accuracies, incorrect_samples)
return losses, accuracies, incorrect_samples
def run_task(data_directory, task_id):
"""
Parse data, build model, and run training and testing for a single task.
:param data_directory: Path to train and test data.
:param task_id: Task to evaluate
"""
print("Train and test for task %d ..." % task_id)
# Parse data
train_files = glob.glob('%s/qa%d_*_train.txt' % (data_directory, task_id))
test_files = glob.glob('%s/qa%d_*_test.txt' % (data_directory, task_id))
dictionary = {"nil": 0}
# Story shape: (SENTENCE_SIZE, STORY_SIZE, NUM_STORIES)
# Questions shape: (14 (see parser.py), NUM_SAMPLES)
# QStory shape: (SENTENCE_SIZE, NUM_SAMPLES)
train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
test_story, test_questions, test_qstory = parse_babi_task(test_files, dictionary, False)
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory, model, loss,
general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss, general_config)
test(test_story, test_questions, test_qstory, memory, model, loss, general_config)
def main(_):
# try:
sys.stdout.flush()
if params.self_test:
self_test()
elif params.decode:
decode()
else:
print ('training')
train()
def main(_):
# try:
sys.stdout.flush()
if params.self_test:
self_test()
elif params.decode:
decode()
else:
print('training')
train()
def main(mode: str, epochs: int, weights_name: str):
if (mode == "train"):
print("Train mode was chosen.")
test_split = 0.05
train(epochs=epochs, save_model=True, test_split=test_split)
# Use the weights just trained to run a prediction on the test samples to see how the model performs.
predict(model_file_name="model", test_split=test_split)
elif (mode == "predict"):
print("Predict mode was chosen.")
predict(model_file_name=weights_name, test_split=0.05)
def main(_):
# try:
print('in main function call')
sys.stdout.flush()
if params.self_test:
self_test()
elif params.decode:
decode()
else:
print('training')
train()
def train_model(epochs, batch_size, learning_rate, weight_decay, beta_1,
beta_2, latent_size, momentum, num_workers, build_dir,
experience_name, saving_rate, checkpoint, cpu):
if epochs is None or batch_size is None or learning_rate is None or weight_decay is None:
return
if beta_1 is None or beta_2 is None or latent_size is None or momentum is None or num_workers is None or build_dir is None:
return
if experience_name is None or saving_rate is None:
return
train(epochs, batch_size, learning_rate, weight_decay, beta_1, beta_2,
latent_size, momentum, num_workers, build_dir, experience_name,
saving_rate, None if checkpoint is None else checkpoint,
False if not cpu else True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-sd",
"--stock_data",
required=True,
help="Specify the path to your stock data")
parser.add_argument("-td",
"--twitter_data",
required=True,
help="Specify the path to your Twitter data")
parser.add_argument("-v",
"--verbose",
required=False,
help="print verbose output",
action='store_true')
parser.add_argument("-ns",
"--no_sentiment",
required=False,
help="train without sentiment",
action='store_true')
args = parser.parse_args()
stock_path = args.stock_data
twitter_path = args.twitter_data
verbose = args.verbose
no_sentiment = args.no_sentiment
stock_files = stock_preprocess.get_stock_csv_files(stock_path)
twitter_file = os.path.join(twitter_path, os.listdir(twitter_path)[0])
stock_data = get_stock_data(stock_files)
twitter_data = get_twitter_data(twitter_file)
train_data, test_data = create_data(stock_data, twitter_data, no_sentiment)
model = StockModel(14 if no_sentiment else 32)
train(model, train_data)
print(test(model, test_data))
def run_task(data_directory, task_id):
"""
Parse data, build model, and run training and testing for a single task.
:param data_directory: Path to train and test data.
:param task_id: Task to evaluate
"""
print("Train and test for task %d ..." % task_id)
# Parse data
train_files = glob.glob('%s/qa%d_*_train.txt' % (data_directory, task_id))
test_files = glob.glob('%s/qa%d_*_test.txt' % (data_directory, task_id))
dictionary = {"nil": 0}
# Story shape: (SENTENCE_SIZE, STORY_SIZE, NUM_STORIES)
# Questions shape: (14 (see parser.py), NUM_SAMPLES)
# QStory shape: (SENTENCE_SIZE, NUM_SAMPLES)
train_story, train_questions, train_qstory = parse_babi_task(
train_files, dictionary, False)
test_story, test_questions, test_qstory = parse_babi_task(
test_files, dictionary, False)
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory,
model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss,
general_config)
test(test_story, test_questions, test_qstory, memory, model, loss,
general_config)
def main():
if FLAGS.use_wikidata:
file.check_files()
if FLAGS.debug:
print("\n-- DEBUG MODE --\n")
saver = Saver()
train_data, val_data, test_data, raw_doc_list = load_data()
saved_model, model = train(train_data, val_data, saver)
assert model.num_layers == 2
if FLAGS.plot:
number_of_docs = len(train_data.labels)
plot(model, number_of_docs)
with torch.no_grad():
test_loss_model, preds_model = model(train_data.get_pyg_graph(device=FLAGS.device), test_data)
# Classification
eval_res = eval(preds_model, test_data, FLAGS.use_wikidata, True, save=True)
y_true = eval_res.pop('y_true')
y_pred = eval_res.pop('y_pred')
if FLAGS.show_eval:
print("Test...")
pprint(eval_res)
all_saved_models = glob.glob(f"{saver.logdir}/*.pt")
# Iterate over the list of filepaths & remove each file.
for filePath in all_saved_models:
try:
os.remove(filePath)
except:
print("Error while deleting file : ", filePath)
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
BATCH_SIZE = 64
NUM_FEATURES = 8
NUM_CLASSES = 2
SEQ_LEN = 30
from model.cnn_mdl_fn import cnn_model_fn
from model.train import train
if __name__ == '__main__':
train()
if __name__ == "__main__":
parse = argparse.ArgumentParser()
parse.add_argument("--train-csv-path", type=str, default="data/train.csv", help="Training csv path for label file")
parse.add_argument("--valid-csv-path", type=str, default="data/valid.csv", help="Validation csv path for label file")
parse.add_argument("--test-csv-path", type=str, default="data/test.csv", help="Testing csv path for label file")
parse.add_argument("--batch-size", type=int, default=64, help="Batch size of images")
parse.add_argument("--lr", type=float, default=0.005, help="Learning rate")
parse.add_argument("--momentum", type=float, default=0.9, help="Momentum")
parse.add_argument("--gamma", type=float, default=0.8, metavar="M",
help="Learning rate step gamma (default: 0.7)")
opt = parse.parse_args()
train_loader, valid_loader, test_loader = load_dataset(train_csv_path=opt.train_csv_path,
valid_csv_path=opt.valid_csv_path,
test_csv_path=opt.test_csv_path,
bs=opt.batch_size, workers=2, transform=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Initialize model
model = Net()
model.to(device)
# Define hyperparameter, optimizer, loss, scheduler
optimizer = optim.SGD(model.parameters(), lr=opt.lr, momentum=opt.momentum)
loss_fn = nn.CrossEntropyLoss()
# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=opt.gamma)
train(model, device, train_loader, valid_loader, loss_fn, optimizer, epoch=50)
def main():
args, other = get_args()
experiment = args.experiment
anserini_path = args.anserini_path
datasets_path = os.path.join(args.data_path, 'datasets')
if args.mode == 'training':
train(args)
elif args.mode == 'inference':
test(args)
else:
if args.interactive:
# TODO: sync with HiCAL
from utils.query import query_sents, visualize_scores
sentid2text, hits = query_sents(args, K=10)
test(args)
else:
folds_path = os.path.join(args.data_path, 'folds',
'{}-folds.json'.format(args.collection))
qrels_path = os.path.join(args.data_path, 'qrels',
'qrels.{}.txt'.format(args.collection))
topK = int(other[0])
alpha = float(other[1])
beta = float(other[2])
gamma = float(other[3])
test_folder_set = int(other[4])
mode = other[5]
# Divide topics according to fold parameters
train_topics, test_topics, all_topics = [], [], []
with open(folds_path) as f:
folds = json.load(f)
for i in range(0, len(folds)):
all_topics.extend(folds[i])
if i != test_folder_set:
train_topics.extend(folds[i])
else:
test_topics.extend(folds[i])
collection_path = os.path.join(
datasets_path, '{}_sents.csv'.format(args.collection))
predictions_path = os.path.join(args.data_path, 'predictions',
'predict.' + experiment)
top_doc_dict, doc_bm25_dict, sent_dict, q_dict, doc_label_dict = eval_bm25(
collection_path)
score_dict = load_bert_scores(predictions_path, q_dict, sent_dict)
if not os.path.isdir('runs'):
os.mkdir('runs')
if mode == 'train':
for a in np.arange(0.0, alpha, 0.1):
for b in np.arange(0.0, beta, 0.1):
for g in np.arange(0.0, gamma, 0.1):
calc_q_doc_bert(score_dict,
'run.' + experiment + '.cv.train',
train_topics, top_doc_dict,
doc_bm25_dict, topK, a, b, g)
base = 'runs/run.' + experiment + '.cv.train'
os.system(
'{}/eval/trec_eval.9.0.4/trec_eval -M1000 -m map {} {}> eval.base'
.format(anserini_path, qrels_path, base))
with open('eval.base', 'r') as f:
for line in f:
metric, qid, score = line.split('\t')
map_score = float(score)
print(test_folder_set, round(a, 2),
round(b, 2), round(g, 2), map_score)
elif mode == 'test':
calc_q_doc_bert(
score_dict,
'run.' + experiment + '.cv.test.' + str(test_folder_set),
test_topics, top_doc_dict, doc_bm25_dict, topK, alpha,
beta, gamma)
def main(args):
cfg = args.__dict__
train(cfg)
optimizer = Adam(model.parameters())
criterion = CrossEntropyLoss()
train_loader = loader(train_data, BATCH_SIZE)
valid_loader = loader(valid_data, BATCH_SIZE)
N_EPOCHS = 10
CLIP = 1
best_valid_loss = float('inf')
for epoch in range(N_EPOCHS):
start_time = time.time()
train_loss, _ = train(model, train_loader, optimizer, criterion, CLIP)
print(train_loss, _)
exit()
valid_loss = evaluate(model, valid_loader, criterion)
print(train_loss, valid_loss)
exit()
end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
args_ = parser.parse_args()
return args_
if __name__ == '__main__':
# Parse script arguments
args = parse_arguments()
# Parse yaml config file parameters
with open(args.config_file_path) as yaml_file:
config_params = yaml.load(yaml_file, Loader=yaml.FullLoader)
if args.mode == 'train':
logging.basicConfig(filename='training.log', level=logging.INFO, filemode='w')
train.train(data_path_source_dir_=config_params['data_params']['data_dir_path'],
training_params=config_params['training_params'],
model_params=config_params['model_params'])
elif args.mode == 'predict':
logging.basicConfig(filename='prediction.log', level=logging.INFO, filemode='w')
predict.predict(data_path_source_dir_=config_params['data_params']['data_dir_path'],
training_params=config_params['training_params'],
model_params=config_params['model_params'])
else:
raise Exception('Script can only be ran with --mode `train` or `predict`')
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
parser.add_argument('--phone_size', default=67, type=int)
parser.add_argument('--feat-dim', default=1324, type=int)
parser.add_argument('--embedding-size', default=256, type=int)
parser.add_argument('--hidden-size', default=256, type=int)
parser.add_argument('--glu-num-layers',
default=1,
type=int,
help='number of glu layers')
parser.add_argument('--dropout', default=0.1, type=float)
parser.add_argument('--dec_num_block', default=6, type=int)
parser.add_argument('--dec_nhead', default=4, type=int)
parser.add_argument('--seed', default=666, type=int)
parser.add_argument('--use_tfb',
dest='use_tfboard',
help='whether use tensorboard',
action='store_true')
parser.add_argument('--noam-scale', default=1.0, type=float)
parser.add_argument('--noam-warmup-steps', default=25000, type=float)
parser.add_argument('--loss', default="l1", type=str)
parser.add_argument('--use-pos-enc', default=0, type=int)
parser.add_argument('--gradient-accumulation-steps', default=1, type=int)
args = parser.parse_args()
import system_info
system_info.print_system_info()
print(args)
from model.train import train
train(args)
BATCH_SIZE = 1
AUGMENT = True
dataloader = {
'train': trainloader(colab=COLAB, batch_size=BATCH_SIZE, augment=AUGMENT),
'val': validloader(colab=COLAB, batch_size=BATCH_SIZE)
}
net = IM2HI()
if torch.cuda.is_available():
net.cuda()
criterion = torch.nn.L1Loss()
# optimizer = optim.SGD(net.parameters(),
# lr=LR,
# momentum=0.9,
# nesterov=True,
# weight_decay=1e-1)
optimizer = torch.optim.Adam(net.parameters(), lr=2e-5, weight_decay=1e-2)
train(net,
dataloader,
criterion=criterion,
optimizer=optimizer,
num_epochs=200,
model_name='im2height_loss',
comment=comment)
# unet1: valid 0.724712, test 0.6318, batch 16, lr=2e-5, weight decay=1e-2, l1loss
# unet ssim loss test 66.45 batch 16, lr=2e-5, weight decay=1e-2,
# unet l2 loss test 22.37 batch 16 lr=2e-5, weight decay=1e-2
return parser.parse_args()
if __name__== "__main__":
args = parse_args()
if args.write_dataset:
writeDataset()
if args.train_model:
trainModel()
model = model.MyModel()
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
# Add a channels dimension
x_train = x_train[..., tf.newaxis]
x_test = x_test[..., tf.newaxis]
train_ds = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).shuffle(10000).batch(32)
test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)
train.train(model, train_ds, test_ds)
exit(0)
ds_train, train_classes)
dl_train = to.utils.data.DataLoader(ds_train,
batch_size=256,
shuffle=True)
input_dim = np.prod(ds_train.__getitem__(0)[0].shape)
args = []
kwargs = {
'last_layer_size': i,
'lr': 5e-4,
}
model = NN(*args, **kwargs).to(to.device(device))
epochs = 10
print("Training model %d for %d epochs..." % (i, epochs))
loss, acc, scores = train(model,
dl_train,
closed_set_accuracy,
device,
epochs=epochs)
print("Trained with loss %f and accuracy %f" % (loss[-1], acc[-1]))
model_name = "model_%d_" % i
models[model_name + "state_dict"] = model.state_dict()
models[model_name + "args"] = args
models[model_name + "kwargs"] = kwargs
to.save(models, './saves')
if (mode == "load"):
models = to.load('./models', map_location=device)
for i in range(2, 11):
test_classes = range(i)
ds_test = tv.datasets.MNIST(root='./',
train=False,
print_opts(args)
G, D = train(args)
'''
# Train CycleGAN
args_dict = {
'image_size': 32,
'g_conv_dim': 32,
'd_conv_dim': 32,
'init_zero_weights': False,
'num_workers': 0,
'train_iters': 5000,
'X': 'Apple',
'Y': 'Windows',
'lambda_cycle': 0.015,
'lr': 0.0003,
'beta1': 0.3,
'beta2': 0.999,
'batch_size': 32,
'checkpoint_dir': 'checkpoints_cyclegan',
'sample_dir': 'samples_cyclegan',
'load': None,
'log_step': 200,
'sample_every': 200,
'checkpoint_every': 1000,
}
args.update(args_dict)
print_opts(args)
G_XtoY, G_YtoX, D_X, D_Y = train(args)
def main():
args, other = get_args()
experiment = args.experiment
anserini_path = args.anserini_path
datasets_path = os.path.join(args.data_path, 'datasets')
if not os.path.isdir('log'):
os.mkdir('log')
if args.mode == 'training':
train(args)
elif args.mode == 'check_performance':
check_dev_performance(args)
elif args.mode == 'inference':
scores = test(args)
print_scores(scores)
else:
folds_path = os.path.join(anserini_path, 'src', 'main', 'resources',
'fine_tuning', args.folds_file)
qrels_path = os.path.join(anserini_path, 'src', 'main', 'resources',
'topics-and-qrels', args.qrels_file)
topK = int(other[0])
alpha = float(other[1])
beta = float(other[2])
gamma = float(other[3])
test_folder_set = int(other[4])
mode = other[5]
# Divide topics according to fold parameters
train_topics, test_topics, all_topics = [], [], []
with open(folds_path) as f:
folds = json.load(f)
for i in range(0, len(folds)):
all_topics.extend(folds[i])
if i != test_folder_set:
train_topics.extend(folds[i])
else:
test_topics.extend(folds[i])
if args.interactive:
sentid2text = query_sents(args)
test(args) # inference over each sentence
collection_path = os.path.join(
datasets_path, args.collection +
'.csv') if not args.interactive else args.interactive_path
predictions_path = os.path.join(
args.data_path, 'predictions', 'predict.' +
experiment) if not args.interactive else os.path.join(
args.data_path, 'predictions', args.predict_path)
top_doc_dict, doc_bm25_dict, sent_dict, q_dict, doc_label_dict = eval_bm25(
collection_path)
score_dict = load_bert_scores(predictions_path, q_dict, sent_dict)
if args.interactive:
top_rank_docs = visualize_scores(collection_path, score_dict)
with open(os.path.join(args.data_path, 'query_sent_scores.csv'),
'w') as scores_file:
for doc in top_rank_docs[:100]:
scores_file.write('{}\t{}\t{}\t{}\t{}\n'.format(
doc[0], sentid2text[doc[0]], doc[1], doc[2],
'BM25' if doc[3] > 0 else 'BERT'))
for doc in top_rank_docs[-100:]:
scores_file.write('{}\t{}\t{}\t{}\t{}\n'.format(
doc[0], sentid2text[doc[0]], doc[1], doc[2],
'BM25' if doc[3] > 0 else 'BERT'))
if not os.path.isdir('runs'):
os.mkdir('runs')
if mode == 'train':
topics = train_topics if not args.interactive else list(
q_dict.keys())
# Grid search for best parameters
for a in np.arange(0.0, alpha, 0.1):
for b in np.arange(0.0, beta, 0.1):
for g in np.arange(0.0, gamma, 0.1):
calc_q_doc_bert(score_dict,
'run.' + experiment + '.cv.train',
topics, top_doc_dict, doc_bm25_dict,
topK, a, b, g)
base = 'runs/run.' + experiment + '.cv.train'
os.system(
'{}/eval/trec_eval.9.0.4/trec_eval -M1000 -m map {} {}> eval.base'
.format(anserini_path, qrels_path, base))
with open('eval.base', 'r') as f:
for line in f:
metric, qid, score = line.split('\t')
map_score = float(score)
print(test_folder_set, round(a, 2),
round(b, 2), round(g, 2), map_score)
elif mode == 'test':
topics = test_topics if not args.interactive else list(
q_dict.keys())
calc_q_doc_bert(
score_dict,
'run.' + experiment + '.cv.test.' + str(test_folder_set),
topics, top_doc_dict, doc_bm25_dict, topK, alpha, beta, gamma)
else:
topics = all_topics if not args.interactive else list(
q_dict.keys())
calc_q_doc_bert(score_dict, 'run.' + experiment + '.cv.all',
topics, top_doc_dict, doc_bm25_dict, topK, alpha,
beta, gamma)
def main():
args = args_parser()
logs = LogSaver(args)
acc_test, loss_test, acc_train, loss_train = [], [], [], []
# ToDo change this
classes = [i for i in range(args.num_classes)]
distrib = Distribute(args.num_workers, len(classes))
train_data_distribution = copy.deepcopy(
distrib.get_distribution(args.dstr_Train,
args.n_labels_per_agent_Train,
args.sub_labels_Train))
test_data_distribution = copy.deepcopy(
distrib.get_distribution(args.dstr_Test, args.n_labels_per_agent_Test,
args.sub_labels_Test))
print(train_data_distribution, "\n\n TEST DISTRIBUTION",
test_data_distribution)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
fed_trainloaders, fed_testloaders, workers = get_dataloaders(
train_data_distribution, test_data_distribution, args.dataset,
args.train_bs, args.test_bs, args.num_workers)
print("TRAINLOADERs ARE CREATED")
batches = extract_batches_per_worker(fed_trainloaders)
batches_test = extract_batches_per_worker(fed_testloaders)
net = Net(10)
# copy weights
# w_glob = net.state_dict()
net.to(device)
loss_func = nn.CrossEntropyLoss()
for rnd in range(args.rounds):
w_local = {}
n = []
# For now all of the updates are calculated sequentially
for worker in workers:
trainloader = batches[worker]
# Batch size is needed to calculate accuracy
w, loss, acc = train(worker,
net,
trainloader,
loss_func,
args.local_ep,
args.train_bs,
device=device)
# ToDo w -> w.state_dict()
w_local[worker] = w #.state_dict()
n.append(len(trainloader))
loss_train.append(copy.deepcopy(loss))
acc_train.append(copy.deepcopy(acc))
net = federated_avg(w_local)
# w_glob = FedAvg(w_local, n)
# Analog to model distribution
# net.load_state_dict(w_glob)
# Perform tests after global update
for worker in workers:
testloader = batches_test[worker]
loss, acc = test(worker,
net,
testloader,
loss_func,
args.test_bs,
device=device)
print(worker.id, "loss", loss, "acc", acc)
acc_test.append(copy.deepcopy(acc))
loss_test.append(copy.deepcopy(loss))
print("Round", rnd)
#print(acc_train[-1], loss_train[-1], acc_test[-1], loss_test[-1])
logs.add_row(acc_train, loss_train, acc_test, loss_test)
print("End of training")
print(acc_train, "\n\n", type(acc_train))
logs.plot(loss_train, loss_test, np.array(acc_train), np.array(acc_test))
print("Plots are created\n", acc_train, "\n\n", loss_train)
logs.save_model(net)
CLIP = config['CLIP']
optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
criterion = nn.CrossEntropyLoss(ignore_index=TRG_PAD_IDX)
# TRAINING
print('started training model .. \n')
best_valid_loss = float('inf')
best_train_loss = float('inf')
for epoch in range(N_EPOCHS):
start_time = time.time()
train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
valid_loss = evaluate(model, test_iterator, criterion)
end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict(), 'min_val_loss_model.pt')
if train_loss < best_train_loss:
best_train_loss = train_loss
torch.save(model.state_dict(), 'min_train_loss_model.pt')
print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')
print(
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch Voice Filter')
parser.add_argument('-b',
'--base_dir',
type=str,
default='.',
help="Root directory of run.")
parser.add_argument('--checkpoint_path',
type=str,
default=None,
help='Path to last checkpoint')
parser.add_argument('-e',
'--embedder_path',
type=str,
required=True,
help="path of embedder model pt file")
parser.add_argument(
'-m',
'--model',
type=str,
required=True,
help="Name of the model. Used for both logging and saving checkpoints."
)
args = parser.parse_args()
chkpt_path = args.checkpoint_path if args.checkpoint_path is not None else None
pt_dir = os.path.join(args.base_dir, config.log['chkpt_dir'], args.model)
os.makedirs(pt_dir, exist_ok=True)
log_dir = os.path.join(args.base_dir, config.log['log_dir'], args.model)
os.makedirs(log_dir, exist_ok=True)
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(
os.path.join(
log_dir,
'%s-%d.log' % (args.model, time.time()))),
logging.StreamHandler()
])
logger = logging.getLogger()
writer = MyWriter(log_dir)
trainloader = create_dataloader(train=True)
testloader = create_dataloader(train=False)
embedder_pt = torch.load(args.embedder_path)
embedder = SpeechEmbedder().cuda()
embedder.load_state_dict(embedder_pt)
embedder.eval()
model = nn.DataParallel(VoiceFilter())
optimizer = torch.optim.Adam(model.parameters(), lr=config.train['adam'])
audio = Audio()
starting_step = 0
starting_epoch = 1
if chkpt_path is not None:
logger.info("Resuming from checkpoint: %s" % chkpt_path)
checkpoint_file = torch.load(chkpt_path)
model.load_state_dict(checkpoint_file['model'])
starting_epoch = checkpoint_file['epoch']
starting_step = checkpoint_file['step']
else:
logger.info("Starting new training run")
scheduler = StepLR(optimizer, step_size=1, gamma=0.7)
for epoch in range(starting_epoch, config.train['epoch'] + 1):
train(embedder, model, optimizer, trainloader, writer, logger, epoch,
pt_dir, starting_step)
validate(audio, model, embedder, testloader, writer, epoch)
scheduler.step()
starting_step = 0
model_saver(model, pt_dir, config.train['epoch'],
config.train['train_step_pre_epoch'])
lstm_model = lstm.get_lstm(embedding_matrix,
num_classes,
300,
max_seq_len,
l2_weight_decay=0.0001,
lstm_dim=50,
dropout_val=0.3,
dense_dim=32,
add_sigmoid=True)
lstm_hist = train.train(x_train_nn,
y_train_nn,
lstm_model,
batch_size=256,
num_epochs=100,
learning_rate=0.001,
early_stopping_delta=0.0001,
early_stopping_epochs=3,
use_lr_stratagy=True,
lr_drop_koef=0.66,
epochs_to_drop=2)
y_lstm = lstm_model.predict(x_test_nn)
test_label = lstm_model.predict(test_df_seq)
train.save_predictions(test, test_label, target_labels, None)
submission = test[[
"id", 'toxic', 'severe_toxic', 'obscene', 'threat', 'insult',
'identity_hate'
def main():
inputDim = 72
outputDim = 2
args = args_parser()
logs = LogSaver(args)
fed_trainloaders, fed_testloaders, workers, worker_sizes = get_dataloaders(args.data_folder,
logs,
args.dstr_Train,
args.dstr_Test,
args.num_workers,
args.train_bs, args.test_bs)
if torch.cuda.is_available():
torch.set_default_tensor_type(torch.cuda.FloatTensor)
device = torch.device("cuda")
else:
device = torch.device("cpu")
# ToDo: inefficient batch extraction
batches = extract_batches_per_worker(fed_trainloaders)
batches_test = extract_batches_per_worker(fed_testloaders)
net = LinearModel(inputDim, outputDim)
net.to(device)
# If iid - compare to a centralized
if args.dstr_Train == "iid":
net1 = LinearModel(inputDim, outputDim)
net1.to(device)
logs.args.num_workers += 1
# copy weights
# w_glob = net.state_dict()
loss_func = nn.CrossEntropyLoss()
acc_test, loss_test, acc_train, loss_train = [], [], [], []
for rnd in range(args.rounds):
w_local = {}
#n = []
# For now all of the updates are calculated sequentially
for worker in workers:
trainloader = batches[worker]
# Batch size is needed to calculate accuracy
w, loss, acc = train(worker, net, trainloader, loss_func, args.local_ep, args.train_bs,
device=device,
lr=args.lr,
optim=args.optimizer)
w_local[worker] = w.state_dict()
loss_train.append(copy.deepcopy(loss))
acc_train.append(copy.deepcopy(acc))
w_glob = FedAvg(list(w_local.values()), worker_sizes)
# Analog to model distribution
net.load_state_dict(w_glob)
#net = federated_avg(w_local)
# Perform tests after global update
# If the test subset is the same for everyone - perform only one test
if args.dstr_Test == "same":
testloader = batches_test[worker]
loss, acc = test(worker, net, testloader, loss_func, args.test_bs, device=device)
acc_test.append(copy.deepcopy(acc))
loss_test.append(copy.deepcopy(loss))
else:
for worker in workers:
testloader = batches_test[worker]
loss, acc = test(worker, net, testloader, loss_func, args.test_bs, device=device)
acc_test.append(copy.deepcopy(acc))
loss_test.append(copy.deepcopy(loss))
# If iid - take last worker and perform a centralized training
if args.dstr_Train == "iid":
centr_worker = workers[0]
print("worker", centr_worker)
centr_train_loader = batches[centr_worker]
net1, loss, acc = train(centr_worker, net1, centr_train_loader, loss_func, args.local_ep, args.train_bs, device=device)
print("Centralized train loss", loss)
loss_train.append(copy.deepcopy(loss))
acc_train.append(copy.deepcopy(acc))
centr_test_loader = batches_test[centr_worker]
loss, acc = test(centr_worker, net1, centr_test_loader, loss_func, args.test_bs, device=device)
acc_test.append(copy.deepcopy(acc))
loss_test.append(copy.deepcopy(loss))
print("Centralized test loss", loss)
print("Round", rnd)
logs.add_row(acc_train, loss_train, acc_test, loss_test)
print("End of training")
print(acc_train, "\n\n", type(acc_train))
print("Plots are created\n", acc_train, "\n\n", loss_train)
logs.plot(loss_train, loss_test, np.array(acc_train), np.array(acc_test))
logs.save_model(net)
from model.train import train
from model.im2hi import IM2HI
from model.dataloader import trainloader, validloader
colab = True
batch_size = 16
dataloader = {
'train': trainloader(colab=colab, batch_size=batch_size),
'val': validloader(colab=colab, batch_size=batch_size)
}
net = IM2HI()
train(net, dataloader, learning_rate=1e-6)
time_str = time.strftime("%m%d-%H%M", time.localtime(time.time()))
rootdir = "{}/{}/{}-semi-{}-fixed-{}-ratio-{}-lr-{}/".format(
"/data/yangy/data_prepare/result", hp['dataname'], time_str,
str(hp['semi']), str(hp['fixed']), str(hp['ratio']), str(args.lr))
os.makedirs(rootdir, exist_ok=True)
hp['rootdir'] = rootdir
np.save('{}parameter.npy'.format(rootdir), hp)
# 获取模型
my_models = load_model(hp)
#获取数据
train_data, test_data = load_data(hp)
#预训练模型
#my_models = pre_train(hp, my_models, train_data, test_data)
# 预训练结果
#result = test(test_data,hp,my_models,'pretrain')
# 训练模型
my_models = train(hp, my_models, train_data)
# 保存模型
save_model(my_models, rootdir)
# 测试模型
result = test(test_data, hp, my_models, 'final')
), batch_size=1000, shuffle=True)
visualize_dataset_tensorboard(train_loader, writer)
lr = 0.01
epochs = 1
model = Net()
# model_state_dict = torch.load(str(URLs.RESULTS_PATH / 'model.pth'))
# model.load_state_dict(model_state_dict)
opt = optim.SGD(model.parameters(), lr, momentum=0.5)
# opt_state_dict = torch.load(str(URLs.RESULTS_PATH / 'optimizer.pth'))
# opt.load_state_dict(opt_state_dict)
train_losses = []
train_counter = []
test_losses = []
test_counter = [i * len(train_loader.dataset) for i in range(epochs + 1)]
device = torch.device("cpu")
if torch.cuda.is_available():
device = torch.device('cuda:0')
model.cuda()
start_time = time.time()
test(model, test_loader, device, test_losses)
for epoch in range(1, epochs + 1):
train(epoch, model, opt, train_loader, device, train_losses, train_counter, 10, writer)
test(model, test_loader, device, test_losses)
writer.close()
print(f'Running time: {time.strftime("%M:%S", time.gmtime(time.time() - start_time))} sec')
plot_perf(train_counter, train_losses, test_counter, test_losses)
# build model
log_string(log, 'compiling model...')
model = GMAN(SE, args, bn_decay=0.1)
loss_criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), args.learning_rate)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=args.decay_epoch,
gamma=0.9)
parameters = count_parameters(model)
log_string(log, 'trainable parameters: {:,}'.format(parameters))
if __name__ == '__main__':
start = time.time()
loss_train, loss_val = train(model, args, log, loss_criterion, optimizer,
scheduler)
plot_train_val_loss(loss_train, loss_val, 'figure/train_val_loss.png')
trainPred, valPred, testPred = test(args, log)
end = time.time()
log_string(log, 'total time: %.1fmin' % ((end - start) / 60))
log.close()
trainPred_ = trainPred.numpy().reshape(-1, trainY.shape[-1])
trainY_ = trainY.numpy().reshape(-1, trainY.shape[-1])
valPred_ = valPred.numpy().reshape(-1, valY.shape[-1])
valY_ = valY.numpy().reshape(-1, valY.shape[-1])
testPred_ = testPred.numpy().reshape(-1, testY.shape[-1])
testY_ = testY.numpy().reshape(-1, testY.shape[-1])
# Save training, validation and testing datas to disk
l = [trainPred_, trainY_, valPred_, valY_, testPred_, testY_]
name = ['trainPred', 'trainY', 'valPred', 'valY', 'testPred', 'testY']
