def train_model(dataset, transform_type):
"""
Train specific model on given dataset.
:param dataset:
:param transform_type:
"""
print('Training model ({}) on {}...'.format(transform_type, dataset))
(X_train, Y_train), (X_test, Y_test) = load_data(dataset)
nb_examples, img_rows, img_cols, nb_channels = X_train.shape
nb_classes = Y_train.shape[1]
input_shape = (img_rows, img_cols, nb_channels)
X_train = transform(X_train, transform_type)
model_name = 'model-{}-cnn-{}'.format(dataset, transform_type)
require_preprocess = False
if (dataset == DATA.cifar_10):
require_preprocess = True
# train
model = models.create_model(dataset, input_shape, nb_classes)
models.train(model, X_train, Y_train, model_name, require_preprocess)
# save to disk
models.save_model(model, model_name)
# evaluate the new model
X_test = transform(X_test, transform_type)
loaded_model = models.load_model(model_name)
scores = loaded_model.evaluate(X_test, Y_test, verbose=2)
print('*** Evaluating the new model: {}'.format(scores))
del loaded_model
def train_composition(dataset, transformation_list):
"""
Train a model on dataset on which a sequence of transformations applied
:param dataset: the original dataset
:param transformation_list: the sequence of transformations
:return:
"""
# Apply a sequence of transformations
(X_train, Y_train), (X_test, Y_test) = load_data(dataset)
X_train = transform(X_train, transformation_list)
nb_examples, img_rows, img_cols, nb_channels = X_train.shape
nb_classes = Y_train.shape[1]
input_shape = (img_rows, img_cols, nb_channels)
# Train a model and save
model_name = 'model-{}-cnn-{}'.format(dataset, 'composition')
require_preprocess = (dataset == DATA.cifar_10)
model = models.create_model(dataset, input_shape, nb_classes)
models.train(model, X_train, Y_train, model_name, require_preprocess)
# save to disk
models.save_model(model, model_name)
# evaluate the new model
loaded_model = models.load_model(model_name)
X_test = transform(X_test, transformation_list)
if require_preprocess:
X_test = normalize(X_test)
scores = loaded_model.evaluate(X_test, Y_test, verbose=2)
print('*** Evaluating the new model: {}'.format(scores))
del loaded_model
def run(task):
"""Run the train/predict flow for `task`."""
st.markdown(f'<h1 align="center">{task}</h1>', unsafe_allow_html=True)
train_button = st.sidebar.button("Train")
sidebar_train_message = st.sidebar.empty()
main_train_message = st.empty()
slug = slugify(task)
trained = is_trained(slug)
if trained:
sidebar_train_message.success(TRAINED_MODEL_MESSAGE)
else:
sidebar_train_message.warning(NO_TRAINED_MODEL_MESSAGE)
main_train_message.warning(NO_TRAINED_MODEL_MESSAGE)
if train_button:
sidebar_train_message.info(TRAINING_MESSAGE)
main_train_message.info(TRAINING_MESSAGE)
train(slug, sidebar_train_message, TRAINING_MESSAGE)
sidebar_train_message.success(TRAINED_MODEL_MESSAGE)
main_train_message.empty()
trained = is_trained(slug)
if trained:
show_metrics = st.sidebar.checkbox("Show metrics")
if show_metrics:
metrics = get_metrics(slug)
for key, value in metrics.items():
st.sidebar.text(f"{key}: {value}")
user_input = st.text_area("Input")
st.text("Output")
if user_input:
output = predict(slug, user_input)
display_function = DISPLAY_FUNCTIONS[slug]
display_function(output)
def start():
print __doc__
regressors = ['Ridge', 'RandomForestRegressor', 'GradientBoostingRegressor'] #! see config.py
features = ['user', 'business', 'checkin', 'votes']
models.train(modelnames=regressors, features=features, limit=2000, stemmer_type="RegexpStemmer",
standardized=False, predict=True, plot=True)
def main(args):
data = np.load(args.dataset, mmap_mode='r')
cut = int(0.9 * len(data))
idx = np.arange(len(data))
np.random.shuffle(idx)
models.train(
args, theanets.Autoencoder,
[data[idx[:cut]]], [data[idx[cut:]]],
monitors={'hid1:out': (0.01, 0.99)})
def main(args):
data = np.load(args.dataset, mmap_mode='r')
cut = int(0.9 * len(data))
models.train(
args, theanets.recurrent.Autoencoder,
theanets.recurrent.batches([data[:cut]], args.time_steps, args.batch_size),
theanets.recurrent.batches([data[cut:]], args.time_steps, args.batch_size),
monitors={'hid1:out': (0.01, 0.99)},
)
def main():
train_data = pd.read_csv('data/train.csv')
bst_data = pd.read_csv('data/bts.csv')
train_data = shuffle(train_data)
train, test = train_test_split(train_data, test_size=0.1)
train = train.dropna()
test = pd.read_csv('data/LocTest.csv')
test = test.dropna()
train_points = train.iloc[:, 1:3].values
train_path_loss = train.iloc[:, 3:9].values
bst_points = bst_data.iloc[:, 1:3].values
location_points = np.concatenate((train_points, bst_points), axis=0)
latitudes, longitudes = grid.build_location_grid(location_points)
bts_coordinates = grid.find_bts_coordinates(bst_points, latitudes,
longitudes)
print("location grid with size [{:d}, {:d}] calculated.".format(
latitudes.shape[0], longitudes.shape[0]))
knn = KNeighborsRegressor(n_neighbors=5)
trained_models = models.train(train_points, train_path_loss, knn)
fingerprints = fingerprint.get_grids(trained_models, latitudes, longitudes,
bts_coordinates)
locate.predict_test_locations(fingerprints, latitudes, longitudes, test,
bst_points)
def train_model(data, model_name):
X, Y = data
transformation_type = TRANSFORMATION.clean
model = models.create_model(DATA.CUR_DATASET_NAME)
print('Training model [{}]...'.format(model_name))
model = models.train(model, X, Y, model_name)
print('Saving model...')
models.save_model(model, model_name)
print('Done.')
return model
def train_model(data, transformation_type=TRANSFORMATION.clean):
X, Y = data
print('Transforming training data set [{}]...'.format(transformation_type))
X = transform(X, transformation_type)
model_name = 'model-{}-cnn-{}'.format(DATA.CUR_DATASET_NAME,
transformation_type)
model = models.create_model(DATA.CUR_DATASET_NAME)
print('Training model [{}]...'.format(model_name))
model = models.train(model, X, Y, model_name)
print('Saving model...')
models.save_model(model, model_name)
print('Done.')
return model
def main(args):
X = np.load(args.dataset, mmap_mode='r')
cut = int(0.9 * len(X))
D = np.load(args.codebook, mmap_mode='r')
print('D:', D.shape)
K = sklearn.metrics.pairwise.euclidean_distances(
X, D, (D * D).sum(axis=1), squared=True).argmin(axis=1)
def batch(Z):
def create():
inputs = np.zeros((args.batch_size, args.time_steps, len(D)), 'f')
outputs = np.zeros((args.batch_size, args.time_steps), 'i')
for b in range(args.batch_size):
o = np.random.randint(len(Z) - args.time_steps - 1)
inputs[b, :, Z[o:o+args.time_steps]] = 1
outputs[b] = Z[o+1:o+1+args.time_steps]
return [inputs, outputs]
return create
net = models.train(
args, theanets.recurrent.Classifier, batch(K), batch(K))
speak = D[list(net.predict_sequence(K[cut:cut+200], 1000))]
np.save('/tmp/speak.npy', speak)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("dataset",
type=str,
choices=['20_simplified', '6_simplified'],
help="Dataset selected.")
parser.add_argument("pretrained_model",
type=str,
default="bert-base-uncased",
help="Petrained natural language model selected.")
parser.add_argument(
"--output_dir",
type=str,
default="./saved_models",
help=
"The output directory where the results and checkpoints will be written."
)
parser.add_argument("--batch_size",
type=int,
default=64,
help="Batch size.")
parser.add_argument(
"--max_length",
type=int,
default=128,
help="The maximum total input sequence length after tokenization."
) # check for each dataset
parser.add_argument("--learning_rate",
type=float,
default=5e-5,
help="The initial learning rate for Adam.")
parser.add_argument("--adam_epsilon",
type=float,
default=1e-8,
help="Epsilon for Adam optimizer.")
parser.add_argument("--weight_decay",
type=float,
default=0.01,
help="Weight decay if we apply some.")
parser.add_argument("--num_train_epochs",
type=int,
default=5,
help="Total number of training epochs to perform.")
parser.add_argument("--num_warmup_steps",
type=int,
default=0,
help="Linear warmup over warmup_steps.")
parser.add_argument("--max_grad_norm",
type=float,
default=1.0,
help="Max gradient norm.")
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
parser.add_argument("--lowercase",
action='store_true',
help="Uses an uncased model.")
parser.add_argument("--lstm_hidden_dim",
type=int,
default=1024,
help="Number of LSTM units.")
parser.add_argument("--dense_hidden_dim",
type=int,
default=32,
help="Number oh dense units.")
parser.add_argument("--num_lstm_layers",
type=int,
default=1,
help="Number of LSTM layers.")
parser.add_argument("--num_dense_layers",
type=int,
default=2,
help="Number of dense layers.")
parser.add_argument("--size_segment",
type=int,
default=200,
help="Size segment.")
parser.add_argument("--size_shift",
type=int,
default=50,
help="Size shift.")
parser.add_argument("--lr_lstm",
type=float,
default=0.001,
help="Learning rate of LSTM.")
parser.add_argument(
"--epochs_decrease_lr_lstm",
type=int,
default=3,
help=
"Number of epoch to descrease learning rate when the validation loss does not improve."
)
parser.add_argument("--reduced_factor_lstm",
type=float,
default=0.95,
help="Reduced factor for learning rate of the LSTM.")
args = parser.parse_args()
dataset2num_labels = {
'20_simplified': 20,
'6_simplified': 6,
}
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
use_token_type_ids = True
conf_name = (
f"{args.num_train_epochs}_{args.batch_size}_{args.max_length}_{args.learning_rate}_"
f"{args.num_warmup_steps}_{args.adam_epsilon}_{args.weight_decay}_{args.max_grad_norm}_{args.seed}_"
f"{args.lstm_hidden_dim}_{args.dense_hidden_dim}_{args.num_lstm_layers}_{args.num_lstm_layers}"
f"{args.lr_lstm}_{args.epochs_decrease_lr_lstm}_{args.reduced_factor_lstm}"
)
num_classes = dataset2num_labels[args.dataset]
model_classifier = RoBert(
num_classes,
bert_pretrained=args.pretrained_model,
device=device,
lstm_hidden_dim=args.lstm_hidden_dim,
dense_hidden_dim=args.dense_hidden_dim,
num_lstm_layers=args.num_lstm_layers,
num_dense_layers=args.num_dense_layers,
)
model_tokenizer = model_classifier.tokenizer()
X_train, y_train, num_segments_train, X_test, y_test, num_segments_test, label2id = load_20newsgroup_segments(
args.dataset,
max_length=args.max_length,
size_segment=args.size_segment,
size_shift=args.size_shift)
train_set = generate_dataset_20newsgroup_segments(
X_train,
y_train,
num_segments_train,
model_tokenizer,
max_length=args.size_segment)
validation_set = generate_dataset_20newsgroup_segments(
X_test,
y_test,
num_segments_test,
model_tokenizer,
max_length=args.size_segment)
model_path = os.path.join(args.output_dir, args.dataset,
args.pretrained_model, conf_name)
if os.path.exists(model_path):
shutil.rmtree(model_path)
os.makedirs(model_path)
id2label = {value: key for (key, value) in label2id.items()}
train(model_classifier,
train_set,
validation_set,
device,
model_path,
id2label,
batch_size=args.batch_size,
weight_decay=args.weight_decay,
num_train_epochs=args.num_train_epochs,
lr=args.learning_rate,
eps=args.adam_epsilon,
num_warmup_steps=args.num_warmup_steps,
max_grad_norm=args.max_grad_norm,
seed=args.seed,
use_token_type_ids=use_token_type_ids,
lr_lstm=args.lr_lstm,
epochs_decrease_lr_lstm=args.epochs_decrease_lr_lstm,
reduced_factor_lstm=args.reduced_factor_lstm)
model_path = 'tmp/' + 'adagvae' + experiment_name + experiment_id
labels = ["recon_1", "recon_2", "kl_1", "kl_2"]
train_data, test_data = dsprites.get_dsprites(train_size=737280,
test_size=737280,
batch_size=64)
vae = AdaGVAE(n_channels=1)
opt = optim.Adam(vae.parameters(), lr=0.0001, betas=(0.9, 0.999), eps=1e-8)
writer = SummaryWriter(log_dir=model_path)
for epoch in range(29):
models.train(vae,
train_data,
epoch,
opt,
verbose=True,
writer=writer,
metrics_labels=labels)
_, metrics = models.test(vae,
test_data,
verbose=True,
metrics_labels=labels,
writer=writer)
with torch.no_grad():
num_samples = 15
x1, x2, _ = next(iter(test_data))
x1 = x1[:15]
x2 = x2[:15]
# to wait for torchsummary to be fixed first.
if CHECKPOINT is not None:
model.load_state_dict(torch.load(CHECKPOINT))
if (T_BAND_GROUP != 0): # KD is used
try:
train(
model,
optimizer,
loss,
train_loader,
hyperparams["epoch"],
scheduler=hyperparams["scheduler"],
device=hyperparams["device"],
supervision=hyperparams["supervision"],
val_loader=val_loader,
display=viz,
name=NAME,
t_net=t_model,
t_alpha=T_KD_ALPHA,
t_temp=T_KD_TEMP,
n_bands=N_BANDS,
)
except KeyboardInterrupt:
# Allow the user to stop the training
pass
probabilities = test(model, img[:, :, :N_BANDS], hyperparams)
prediction = np.argmax(probabilities, axis=-1)
def test_fc_100_100_10_accuracy(mnist):
model = fc_100_100_10()
X_train, y_train, X_test, y_test = mnist
train(model, X_train, y_train, epochs=2)
assert isclose(accuracy(model, X_test, y_test), 0.544, abs_tol=0.01)
def main(args):
# Main function flow
# 0. Load experiment conditions
config_list, exp_name = _load_params(args.file_path)
if args.single_mode:
exp_name = exp_name + "_single"
# 1. Create Agents
agents, core_agent = create_agents(config_list)
# # 2. Create Environments
envs, core_env = create_envs(agents, core_agent, exp_name)
# 2.5 Create directory for save temporally
create_directory(agents, core_agent, exp_name)
if not args.single_mode:
# 3. Train model
# best_agent = models.train(envs, agents, core_env, core_agent, core_agent.CONSTANTS.N_EPISODE, len(agents), exp)
exp = hyper_dash_settings(exp_name)
models.train(envs, agents,
core_env, core_agent, core_agent.CONSTANTS.N_EPISODE,
len(agents), exp, exp_name)
exp.end()
# torch.save(best_agent.policy_net, best_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH + "/dqn_pong_model")
for agent in agents:
with open(
core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
"/{}/internal-agent/{}.pkl".format(
exp_name, agent.get_name()), 'wb') as f:
cloudpickle.dump(agent.policy_net, f)
with open(
core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
"/{}/core-agent/{}.pkl".format(
exp_name, core_agent.get_name()), 'wb') as f:
cloudpickle.dump(core_agent.policy_net, f)
else:
# 3.Train model
exp = hyper_dash_settings(exp_name)
models.single_train(envs, agents, core_env,
core_agent, core_agent.CONSTANTS.N_EPISODE,
len(agents), exp, exp_name)
exp.end()
with open(
core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
"/{}/core-agent/{}.pkl".format(
exp_name, core_agent.get_name()), 'wb') as f:
cloudpickle.dump(core_agent.policy_net, f)
# 4. Test model
del agents
test_env = create_test_envs(core_agent, exp_name)
with open(
core_agent.CONSTANTS.OUTPUT_DIRECTORY_PATH +
"/{}/core-agent/{}.pkl".format(exp_name, core_agent.get_name()),
'rb') as f:
policy_net = cloudpickle.load(f)
policy_net.eval()
exp_test = hyper_dash_settings(exp_name + "_test")
models.test(test_env,
1,
policy_net,
exp_test,
exp_name,
render=False,
agent=core_agent)
exp_test.end()
os.system('pause')
# ---------------中断程序------------------
if CHECKPOINT is not None:
model.load_state_dict(torch.load(CHECKPOINT))
# 训练模型！！！
try:
train(model,
optimizer,
loss,
train_loader,
hyperparams['epoch'],
scheduler=hyperparams['scheduler'],
device=hyperparams['device'],
supervision=hyperparams['supervision'],
val_loader=val_loader,
display=viz,
img=img,
gt=gt,
hyperparams=hyperparams,
n_classes=N_CLASSES,
viz=viz,
LABEL_VALUES=LABEL_VALUES) # for metric
except KeyboardInterrupt:
# Allow the user to stop the training to do inference
pass
# 对整个数据集计算了预测，而不仅仅是对test做预测
probabilities = test(model, img, hyperparams)
prediction = np.argmax(probabilities, axis=-1)
model = NeuralNetwork(**nn_params)
model = model.to(model.device)
optimizer = optim.Adam(model.parameters(), **opt_params)
loss_fn = torch.nn.CrossEntropyLoss(reduction="elementwise_mean")
# Create callbacks
checkpoint = CheckPoint(model, "modelA.ckpt")
earlystop = EarlyStopping(**earlystop_params)
# Train and evaluate the model
flg_stop = False
for epoch in range(1, params["n_epochs"] + 1):
print("\n[EPOCH %d]" % (epoch))
loss_train = train(model,
trainA_loader,
optimizer,
loss_fn,
epoch,
description="Train on task A")
print()
loss_test, acc_test = test(model,
testA_loader,
loss_fn,
description="Test on task A")
print()
# Callbacks
checkpoint.backup(loss_test)
flg_stop = earlystop.check(loss_test)
if flg_stop:
break
print("------------------------------------------------------------------")
for input, _ in train_loader:
break
summary(model.to(hyperparams['device']), input.size()[1:])
# ---------------中断程序------------------
# print([DATASET + ' ' + MODEL])
os.system('pause')
# ---------------中断程序------------------
if CHECKPOINT is not None:
model.load_state_dict(torch.load(CHECKPOINT))
# 训练模型！！！
try:
train(model, optimizer, loss, train_loader, hyperparams['epoch'],
scheduler=hyperparams['scheduler'], device=hyperparams['device'],
supervision=hyperparams['supervision'], val_loader=val_loader,
display=viz)
except KeyboardInterrupt:
# Allow the user to stop the training to do inference
pass
# 对整个数据集计算了预测，而不仅仅是对test做预测
probabilities = test(model, img, hyperparams)
prediction = np.argmax(probabilities, axis=-1)
# 2020.4.20增加，多次获取metrics
# run_results = metrics(prediction, test_gt, ignored_labels=hyperparams['ignored_labels'], n_classes=N_CLASSES)
run_results = metrics(prediction, gt, ignored_labels=hyperparams['ignored_labels'], n_classes=N_CLASSES)
mask = np.zeros(gt.shape, dtype='bool')
PREFIX = os.environ.get('PREFIX', '.')
X_train, y_train, X_test, y_test = mnist()
if not args.random_seed:
K.clear_session()
tf.set_random_seed(1234)
np.random.seed(1234)
no_defense_model = fc_100_100_10()
print(f"Training {no_defense_model.name}...")
train(no_defense_model,
X_train,
y_train,
args.epochs,
verbose=True,
stop_on_stable_weights=True,
reduce_lr_on_plateau=True,
stop_on_stable_weights_patience=60,
reduce_lr_on_plateau_patience=30)
print(f"Saving {no_defense_model.name}...")
save_to_file(no_defense_model, PREFIX)
for n_components in args.n_components:
if args.pca:
pca = cached(f"pca-{n_components}")
filtered_model = pca_filtered_model(no_defense_model,
X_train,
n_components,
pca=pca)
args = argument_parser.parse_args()
X_train, y_train, X_test, y_test = mnist()
models_win_counter = {
"reconstructed_model": 0,
"retrained_model": 0,
"reretrained_model": 0,
}
for n_components in [784, 331, 100, 80, 60, 20]:
reconstructed_model = pca_filtered_model(fc_100_100_10(), X_train,
n_components)
train(reconstructed_model,
X_train,
y_train,
epochs=-1,
stop_on_stable_weights=True)
retrained_model = pca_filtered_model(fc_100_100_10(), X_train,
n_components)
X_train = retrained_model.preprocessing_fn(X_train)
train(retrained_model,
X_train,
y_train,
epochs=-1,
stop_on_stable_weights=True)
best_model = compare_two_models(reconstructed_model, retrained_model)
reretrained_model = pca_filtered_model(fc_100_100_10(), X_train,
import models
import preprocess_data
import sys
symbols = sys.argv[1:] #loads symbols from the command line.
for symbol in symbols: #runs the models on every symbol.
df = preprocess_data.loadDataset(symbol)
scores_models = models.train(df)
print(
f"Scores of models on the dataset {symbol} (Linear, Poly, RBF, Sigmoid, RFC, AdaBoost, VotingClassifier): {[score * 100 for score in scores_models]}"
)
# using gpu
parser.add_argument('--gpu',
action="store_true",
help='add to train on gpu',
default=False,
dest='train_on_gpu')
# retrieve the results
results = parser.parse_args()
data_dir = results.data_dir
learning_rate = results.learning_rate
epochs = results.epochs
train_on_gpu = results.train_on_gpu
hidden_units = results.hidden_units
arch = results.arch
batch_size = results.batch_size
save_dir = results.save_dir
optim = results.optim
models.train(data_dir,
lr=learning_rate,
train_on_gpu=train_on_gpu,
epochs=epochs,
hidden_units=hidden_units,
arch=arch,
batch_size=batch_size,
save_dir=save_dir,
optim=optim)
def test_pca_filtered_keeping_10_components_accuracy(mnist):
X_train, y_train, X_test, y_test = mnist
model = pca_filtered_model(fc_100_100_10(), X_train, 10)
train(model, X_train, y_train, epochs=2)
assert isclose(accuracy(model, X_test, y_test), 0.44, abs_tol=0.01)
def trainBoostedClassifier(self, classifier, level=0):
milestones = self.config['milestones']
lr = self.config['lr']
if self.resume_epoch != -1:
self.classifiers = self.classifiers[:-1]
self.weights = self.weights[:-1]
start = self.resume_epoch
tmp = -1
for m in range(len(milestones)):
if milestones[m] <= self.resume_epoch:
lr = lr * self.config['gamma']
tmp = m
else:
break
if tmp != -1:
milestones = milestones[tmp:]
milestones = list(np.array(milestones) - self.resume_epoch)
else:
start = 0
id_classifier = len(self.classifiers)
print(level * " " + "Training Boosted Classifier n°" +
str(id_classifier) + "...")
optimizer = optim.SGD(classifier.parameters(),
lr=lr,
momentum=self.config['momentum'],
weight_decay=self.config['weight_decay'])
scheduler = lr_scheduler.MultiStepLR(optimizer,
milestones=milestones,
gamma=self.config['gamma'],
last_epoch=-1)
#Adversarial training for the first classifier
if id_classifier == 0:
attack = LinfPGDAttack(classifier,
eps=self.config['eps'] / 255,
eps_iter=self.config['eps_iter'] / 255,
nb_iter=self.config['nb_iter'],
rand_init=self.config['rand_init'],
clip_min=self.config['clip_min'],
clip_max=self.config['clip_max'])
for epoch in range(start, self.config['epochs']):
classifier.train()
models.adversarialTrain(classifier,
self.device,
self.train_loader,
optimizer,
epoch,
attack,
level=level + 1)
scheduler.step()
classifier.eval()
accuracy_under_attack = models.test_under_attack(
classifier,
self.device,
self.test_loader,
attack,
level=level + 1)
accuracy = models.test(classifier,
self.device,
self.test_loader,
level=level + 1)
models.save_model(self.save_dir,
id_classifier,
self.device,
classifier,
accuracy,
accuracy_under_attack,
epoch,
level=level + 1)
models.updateAndSaveBestAccuracies(self.save_dir,
id_classifier,
self.device,
classifier,
accuracy,
accuracy_under_attack,
level=level + 1)
else: #Natural training on the adversarial data set created against the mixture
adversarial_train_loader, adversarial_test_loader = self.adversarialTrainLoader(
level=level + 1)
for epoch in range(start, self.config['epochs']):
classifier.train()
models.train(classifier,
self.device,
adversarial_train_loader,
optimizer,
epoch,
level=level + 1)
scheduler.step()
classifier.eval()
accuracy_under_attack = models.test(classifier,
self.device,
adversarial_test_loader,
level=level + 1)
accuracy = models.test(classifier,
self.device,
self.test_loader,
level=level + 1)
models.save_model(self.save_dir,
id_classifier,
self.device,
classifier,
accuracy,
accuracy_under_attack,
epoch,
level=level + 1)
models.updateAndSaveBestAccuracies(self.save_dir,
id_classifier,
self.device,
classifier,
accuracy,
accuracy_under_attack,
level=level + 1)
classifier, acc, top_acc_under_attack = models.load_model(
self.save_dir,
id_classifier,
-1,
self.device,
self.config['number_of_class'],
top_acc_under_attack=True)
self.classifiers.append(classifier)
#%%
data, label = load_data(data_path, label_path, 'indian_pines')
#%%
get_value_data(data, label)
#%%
DATA = pd.read_csv('datasets/Indian_pines.csv', header=None).values
data_D = DATA[:, :-1]
data_L = DATA[:, -1]
data_train, data_test, label_train, label_test = train_test_split(
data_D, data_L, test_size=0.8)
#%%
train_set = GetLoader(data_train, label_train)
train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
val_set = GetLoader(data_test, label_test)
val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False)
#%%
data_p, label_p = next(iter(train_loader))
# print(data_p[:-1])
#%%
net = Baseline(INPUT_CHANNELS, CLASSES, dropout=False)
optimizer = optim.Adam(net.parameters(), lr=0.0001)
weight = torch.ones(CLASSES)
weight[torch.LongTensor([0])] = 0.
w = weight.to(DEVICE)
criterion = nn.CrossEntropyLoss(weight=w)
#%%
train_loss, val_accuracy = train(net, optimizer, criterion, train_loader,
val_loader, EPOCH, DEVICE)
plot_curve(train_loss)
plot_curve(val_accuracy)
"epochs": -1,
"early_stopping": True,
"reduce_lr_on_plateau": True,
"early_stopping_patience": 60
},
{
"epochs": -1,
"stop_on_stable_weights": True,
"reduce_lr_on_plateau": True,
"reduce_lr_on_plateau_patience": 30
},
]
accuracies_list = []
epochs_list = []
X_train, y_train, X_test, y_test = mnist()
for kwargs in kwargs_list:
model = fc_100_100_10()
history = train(model, X_train, y_train, **kwargs)
n_epochs = len(history.epoch)
test_set_accuracy = accuracy(model, X_test, y_test)
accuracies_list.append(test_set_accuracy)
epochs_list.append(n_epochs)
print("#" * 80)
for kwargs, test_set_accuracy, epochs in zip(kwargs_list, accuracies_list,
epochs_list):
print(f"{kwargs} -> {test_set_accuracy}, trained for {epochs} epochs")
print("#" * 80)
# to find the best parameters of a given model. A parameters grid should be provided.
if finetune:
print("Finetuning ...")
model = models[model_name]['build_fn']
param_grid = models[model_name]['params']
gs, fitted_model, pred = search_pipeline(X_train,
X_test,
y_train,
model,
param_grid,
scoring_fit='accuracy')
best_parameters = get_best_parameters(gs)
y_pred = m.predict(fitted_model, X_test)
accuracy(y_test, y_pred > 0.5)
#results = cross_validation( model, X,y, n_splits=n_sp, **best_parameters)
if train:
print("Training ...")
model = models[model_name]['build_fn']
history = m.train(model,
X_train,
y_train,
X_test,
y_test,
batch_size=batch_size,
epochs=epochs)
print("Cross validation ...")
results = cross_validation(model, X_train, y_train, n_splits=n_sp)
y_pred = m.predict(history, X_test)
accuracy(y_test, y_pred > 0.5)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("dataset",
type=str,
choices=['20_simplified', '6_simplified'],
help="Dataset selected.")
parser.add_argument("model_type",
type=str,
choices=['bert', 'distilbert'],
help="Petrained natural language model type selected.")
parser.add_argument(
"--output_dir",
type=str,
default="./saved_models",
help=
"The output directory where the results and checkpoints will be written."
)
parser.add_argument("--batch_size",
type=int,
default=64,
help="Batch size.")
parser.add_argument(
"--max_length",
type=int,
default=128,
help="The maximum total input sequence length after tokenization."
) # check for each dataset
parser.add_argument("--learning_rate",
type=float,
default=5e-5,
help="The initial learning rate for Adam.")
parser.add_argument("--adam_epsilon",
type=float,
default=1e-8,
help="Epsilon for Adam optimizer.")
parser.add_argument("--weight_decay",
type=float,
default=0.01,
help="Weight decay if we apply some.")
parser.add_argument("--num_train_epochs",
type=int,
default=5,
help="Total number of training epochs to perform.")
parser.add_argument("--num_warmup_steps",
type=int,
default=0,
help="Linear warmup over warmup_steps.")
parser.add_argument("--max_grad_norm",
type=float,
default=1.0,
help="Max gradient norm.")
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
parser.add_argument("--lowercase",
action='store_true',
help="Uses an uncased model.")
args = parser.parse_args()
dataset2num_labels = {
'20_simplified': 20,
'6_simplified': 6,
}
models_use_token_tipe_ids = frozenset({
"bert",
})
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
conf_name = f"{args.num_train_epochs}_{args.batch_size}_{args.max_length}_{args.learning_rate}_{args.num_warmup_steps}_{args.adam_epsilon}_{args.weight_decay}_{args.max_grad_norm}_{args.seed}"
use_token_type_ids = True if args.model_type in models_use_token_tipe_ids else False
num_classes = dataset2num_labels[args.dataset]
model_tokenizer, model_classifier = petrained_language_model_classifier(
args.model_type,
dataset2num_labels[args.dataset],
lowercase=args.lowercase,
)
train_set, validation_set, label2id, _, _ = get_dataset(
model_tokenizer,
args.dataset,
max_length=args.max_length,
seed=args.seed,
use_token_type_ids=use_token_type_ids,
)
model_path = os.path.join(args.output_dir, args.dataset, args.model_type,
conf_name)
if os.path.exists(model_path):
shutil.rmtree(model_path)
os.makedirs(model_path)
id2label = {value: key for (key, value) in label2id.items()}
train(model_classifier,
train_set,
validation_set,
device,
model_path,
id2label,
batch_size=args.batch_size,
weight_decay=args.weight_decay,
num_train_epochs=args.num_train_epochs,
lr=args.learning_rate,
eps=args.adam_epsilon,
num_warmup_steps=args.num_warmup_steps,
max_grad_norm=args.max_grad_norm,
seed=args.seed,
use_token_type_ids=use_token_type_ids)
for input, _ in train_loader:
break
summary(model.to(hyperparams["device"]), input.size()[1:])
# We would like to use device=hyperparams['device'] altough we have
# to wait for torchsummary to be fixed first.
if CHECKPOINT is not None:
model.load_state_dict(torch.load(CHECKPOINT))
try:
train(
model,
optimizer,
loss,
train_loader,
hyperparams["epoch"],
scheduler=hyperparams["scheduler"],
device=hyperparams["device"],
supervision=hyperparams["supervision"],
val_loader=val_loader,
display=viz,
)
except KeyboardInterrupt:
# Allow the user to stop the training
pass
probabilities = test(model, img, hyperparams)
prediction = np.argmax(probabilities, axis=-1)
run_results = metrics(
prediction,
test_gt,
def train_and_test(model_name, compression_type, training_size, test_size):
# train model
if compression_type == "basic" and test_size == 224:
start = time.process_time()
print("training")
model = train(model_name, train_flow, valid_flow)
end = time.process_time()
else:
start = time.process_time()
model = Xception(
weights=None,
include_top=False,
#input_shape=(224,224,3)
)
model = build_model(model)
print("loading weight")
model.load_weights("results/Xception.ckpt")
end = time.process_time()
training_time = end - start
FLOW_MAP = {
"nearest": test_flow_nearest,
"box": test_flow_box,
"lanczos": test_flow_lanczos,
"hamming": test_flow_hamming,
}
test_flow = FLOW_MAP[compression_type]
# save model
# np.savez("model_%s.npz" % model_name, **model)
#Evaluate Model
# get compression specific test flow
# test_flow =
start = time.process_time()
train_pred = model.predict(valid_flow)
train_test = valid_flow.classes
train_accuracy = metrics.accuracy_score(train_test, np.round(train_pred))
# train_accuracy = history.history["accuracy"]
test_pred = model.predict(test_flow)
test_test = test_flow.classes
test_accuracy = metrics.accuracy_score(test_test, np.round(test_pred))
print("Sample predictions: ", np.round(test_pred))
print("Sample actual labels: ", test_test)
test_matrix = metrics.confusion_matrix(test_test,
np.round(test_pred)).ravel()
end = time.process_time()
test_time = end - start
print("Model: %s\n" % model_name)
print("Image Compression: %s\n" % compression)
print("Train Image Size: %d\n" % training_size)
print("Test Image Size: %d\n" % test_size)
print("Train accuracy: %f\n" % train_accuracy)
print("Test accuracy: %f\n" % test_accuracy)
print("True Negatives: %f\n" % test_matrix[0])
print("False Positives: %f\n" % test_matrix[1])
print("False Negatives: %f\n" % test_matrix[2])
print("True Positives: %f\n" % test_matrix[3])
print("Training Time: %s sec\n" % training_time)
print("Testing Time: %s sec\n" % test_time)
# save results
try:
with open("results/%s_results.txt" % model_name, "a") as f:
f.write("Model: %s\n" % model_name)
f.write("Image Compression: %s\n" % compression)
f.write("Train Image Size: %d\n" % training_size)
f.write("Test Image Size: %d\n" % test_size)
f.write("Train accuracy: %f\n" % train_accuracy)
f.write("Test accuracy: %f\n" % test_accuracy)
f.write("True Negatives: %f\n" % test_matrix[0])
f.write("False Positives: %f\n" % test_matrix[1])
f.write("False Negatives: %f\n" % test_matrix[2])
f.write("True Positives: %f\n" % test_matrix[3])
f.write("Training Time: %s sec\n" % training_time)
f.write("Testing Time: %s sec\n" % test_time)
f.write("\n\n")
f.close()
except Exception as e:
print(e)
return {
"Model": model_name,
"Image Compression": compression,
"Train Image Size": training_size,
"Test Image Size": test_size,
"Training Accuracy": train_accuracy,
"Test Accuracy": test_accuracy,
"True Negative": test_matrix[0],
"False Positive": test_matrix[1],
"False Negative": test_matrix[2],
"True Positive": test_matrix[3],
"Train Time": training_time,
"Test Time": test_time
}
export_dir = "/tmp/export/"
export_name = "pet-model" export_version = 2
images, labels = datasets.input_pipeline(dataset_dir, batch_size, is_training=True)
test_images, test_labels = datasets.input_pipeline(dataset_dir, batch_size, is_training=False)
with tf.variable_scope("models") as scope:
logits = nets.inference(images, is_training=True)
scope.reuse_variables()
test_logits = nets.inference(test_images, is_training=False)
total_loss = models.compute_loss(logits, labels)
train_accuracy = models.compute_accuracy(logits, labels)
test_accuracy = models.compute_accuracy(test_logits, test_labels)
global_step = tf.Variable(0, trainable=False)
learning_rate = models.get_learning_rate(global_step, initial_learning_rate, decay_steps, decay_rate)
train_op = models.train(total_loss, learning_rate, global_step, train_vars)
saver = tf.train.Saver(max_to_keep=max_checkpoints_to_keep)
checkpoints_dir = os.path.join(save_dir, datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
if not os.path.exists(save_dir):
os.mkdir(save_dir)
if not os.path.exists(checkpoints_dir):
os.mkdir(checkpoints_dir)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coords = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coords)
with tf.variable_scope("models"):
nets.load_caffe_weights("data/VGG16.npz", sess, ignore_missing=True)
def train(opt):
# Load data
loader = DataLoader(opt)
opt.vocab_size = loader.vocab_size
opt.seq_length = loader.seq_length
# Tensorboard summaries (they're great!)
tb_summary_writer = tb and tb.SummaryWriter(opt.checkpoint_path)
# Load pretrained model, info file, histories file
infos = {}
histories = {}
if opt.start_from is not None:
with open(os.path.join(opt.start_from,
'infos_' + opt.id + '.pkl')) as f:
infos = cPickle.load(f)
saved_model_opt = infos['opt']
need_be_same = ["rnn_type", "rnn_size", "num_layers"]
for checkme in need_be_same:
assert vars(saved_model_opt)[checkme] == vars(
opt
)[checkme], "Command line argument and saved model disagree on '%s' " % checkme
if os.path.isfile(
os.path.join(opt.start_from, 'histories_' + opt.id + '.pkl')):
with open(
os.path.join(opt.start_from,
'histories_' + opt.id + '.pkl')) as f:
histories = cPickle.load(f)
iteration = infos.get('iter', 0)
epoch = infos.get('epoch', 0)
val_result_history = histories.get('val_result_history', {})
loss_history = histories.get('loss_history', {})
lr_history = histories.get('lr_history', {})
#ss_prob_history = histories.get('ss_prob_history', {})
loader.iterators = infos.get('iterators', loader.iterators)
loader.split_ix = infos.get('split_ix', loader.split_ix)
if opt.load_best_score == 1:
best_val_score = infos.get('best_val_score', None)
models = Seq2Seq().cuda()
# Create model
optimizer = utils.build_optimizer_adam(models.parameters(), opt)
update_lr_flag = True
sc_flag = False
while True:
# Update learning rate once per epoch
if update_lr_flag:
# Assign the learning rate
if epoch > opt.learning_rate_decay_start and opt.learning_rate_decay_start >= 0:
frac = (epoch - opt.learning_rate_decay_start
) // opt.learning_rate_decay_every
decay_factor = opt.learning_rate_decay_rate**frac
opt.current_lr = opt.learning_rate * decay_factor
else:
opt.current_lr = opt.learning_rate
utils.set_lr(optimizer, opt.current_lr)
# Assign the scheduled sampling prob
if epoch > opt.scheduled_sampling_start and opt.scheduled_sampling_start >= 0:
frac = (epoch - opt.scheduled_sampling_start
) // opt.scheduled_sampling_increase_every
#opt.ss_prob = min(opt.scheduled_sampling_increase_prob * frac, opt.scheduled_sampling_max_prob)
#model.ss_prob = opt.ss_pro
# Load data from train split (0)
start = time.time()
data = loader.get_batch('train')
data_time = time.time() - start
start = time.time()
# Unpack data
torch.cuda.synchronize()
tmp = [
data['fc_feats'], data['att_feats'], data['labels'], data['dist'],
data['masks'], data['att_masks']
]
tmp = [_ if _ is None else torch.from_numpy(_).cuda() for _ in tmp]
fc_feats, att_feats, labels, dist_label, masks, attmasks = tmp
labels = labels.long()
nd_labels = labels
batchsize = fc_feats.size(0)
# Forward pass and loss
d_steps = 1
g_steps = 1
#print (torch.sum(labels!=0), torch.sum(masks!=0))
if 1:
if 1:
models.train()
optimizer.zero_grad()
wordact = models(
labels.view(batchsize, -1).transpose(1, 0),
labels.view(batchsize, -1).transpose(1, 0), fc_feats)
wordact_t = wordact.transpose(1, 0)[:, 1:, :]
wordact_t = wordact_t.contiguous().view(
wordact_t.size(0) * wordact_t.size(1), -1)
labels_flat = labels.view(batchsize, -1)
wordclass_v = labels_flat[:, 1:]
wordclass_t = wordclass_v.contiguous().view(\
wordclass_v.size(0)*wordclass_v.size(1), -1)
loss_xe = F.cross_entropy(wordact_t[...],
wordclass_t[...].view(-1))
train_loss = loss_xe
train_loss.backward()
optimizer.step()
if 1:
if iteration % opt.print_freq == 1:
print('Read data:', time.time() - start)
if not sc_flag:
print("iter {} (epoch {}), train_loss = {:.3f}, data_time = {:.3f}" \
.format(iteration, epoch, loss_xe, data_time))
else:
print("iter {} (epoch {}), avg_reward = {:.3f}, data_time = {:.3f}, time/batch = {:.3f}" \
.format(iteration, epoch, np.mean(reward[:,0]), data_time, total_time))
# Update the iteration and epoch
iteration += 1
if data['bounds']['wrapped']:
epoch += 1
update_lr_flag = True
# Write the training loss summary
if (iteration % opt.losses_log_every == 0):
add_summary_value(tb_summary_writer, 'train_loss', train_loss,
iteration)
add_summary_value(tb_summary_writer, 'learning_rate',
opt.current_lr, iteration)
#add_summary_value(tb_summary_writer, 'scheduled_sampling_prob', model.ss_prob, iteration)
if sc_flag:
add_summary_value(tb_summary_writer, 'avg_reward',
np.mean(reward[:, 0]), iteration)
loss_history[
iteration] = train_loss if not sc_flag else np.mean(
reward[:, 0])
lr_history[iteration] = opt.current_lr
#ss_prob_history[iteration] = model.ss_prob
# Validate and save model
if (iteration % opt.save_checkpoint_every == 0):
checkpoint_path = os.path.join(
opt.checkpoint_path,
'langmodel{:05d}.pth'.format(iteration))
torch.save(models.state_dict(), checkpoint_path)
optimizer_path = os.path.join(opt.checkpoint_path,
'optimizer.pth')
torch.save(optimizer.state_dict(), optimizer_path)
queries, tokenized_queries, query_terms_inverted, query_term_ids = create_query_resources(
)
inverted_index, tf_C, query_word_positions, unique_terms_per_document, avg_doc_length, document_length, \
collection_length = build_misc_resources(document_ids, query_terms_inverted)
document_term_freqs = inverted_index
create_all_lexical_run_files(index,
document_ids,
queries,
document_term_freqs,
collection_length,
tf_C,
tokenized_queries,
background_model=tf_C,
idf2df=id2df,
num_documents=num_documents)
features = extract_features(queries, document_ids, index,\
document_term_freqs, avg_doc_length, id2df, num_documents, tokenized_queries, collection_length)
datasets, test_features = zip(
*[get_dataset_for_features(features, i=i) for i in range(10)])
test_features = test_features[0]
id2ext, ext2id = get_document_id_maps(index, document_ids)
for i, dataset in enumerate(datasets):
model = train(dataset)
evaluate_model(model, test_features, "./regression", i)
train_data = tenseLoader('train')
train_len = train_data.len()
params.KL_W = 0.0
total_bleu = 0.0
total_loss = 0.0
total_KL = 0.0
bleu_400 = 0.0
loss_400 = 0.0
KL_400 = 0.0
random_list = np.random.permutation(train_len)
for train_i in range(1, train_len + 1): #, train_len+1
input_tensor, input_cond, target_tensor, target_cond = train_data.getPair(
random_list[train_i - 1])
input, output, target, use_TF, bleu4, loss, KLloss = models.train(
input_tensor, target_tensor, input_cond, target_cond, encoder,
decoder, cond_embed, encoder_optimizer, decoder_optimizer,
cond_embed_optimizer)
models.KL_anneal(train_i)
total_bleu += bleu4
total_loss += loss
total_KL += KLloss
bleu_400 += bleu4
loss_400 += loss
KL_400 += KLloss
file_bleu.write('%f\n' % (bleu4))
file_loss.write('%f\n' % (loss))
file_KL.write('%f\n' % (KLloss))
if (train_i % 400 == 0):
print('vocab %d/%d: %s/%s(%s), bleu=%4f, loss=%4f, KL=%4f' %
