def test_acc_last_epoch(model_list,
dataset='mnist',
num_classes=10,
noise_ratio=10,
epochs=50):
"""
Test acc throughout training.
"""
print('Dataset: %s, epochs: %s, noise ratio: %s%%' %
(dataset, epochs, noise_ratio))
# load data
_, _, X_test, Y_test = get_data(dataset)
# convert class vectors to binary class matrices
Y_test = to_categorical(Y_test, num_classes)
# load model
image_shape = X_test.shape[1:]
model = get_model(dataset, input_tensor=None, input_shape=image_shape)
sgd = SGD(lr=0.01, momentum=0.9)
for model_name in model_list:
# the critical sample ratio of the representations learned at every epoch
model_path = 'model/%s_%s_%s.hdf5' % (model_name, dataset, noise_ratio)
model.load_weights(model_path)
model.compile(loss=cross_entropy, optimizer=sgd, metrics=['accuracy'])
_, test_acc = model.evaluate(X_test, Y_test, batch_size=128, verbose=0)
print('model: %s, epoch: %s, test_acc: %s' %
(model_name, epochs - 1, test_acc))
def learn_nn(dataset_name, targets, batch_size, epochs):
for target in targets:
_, X, Y = dt.get_data(target=target, dataset_name=dataset_name)
# Модель
model = md.get_simple_nn(X.shape[1])
if dataset_name == 'full harding':
non_cat_title = tl.full_hard_non_cat_title
cat_title = tl.full_hard_cat_title
elif dataset_name == 'double harding':
non_cat_title = tl.double_hard_non_cat_title
cat_title = tl.double_hard_cat_title
else:
raise ValueError
# Обучение
sc_data = X[non_cat_title]
scaler = preprocessing.StandardScaler()
sc_data = scaler.fit_transform(sc_data)
sc_data = pd.DataFrame(sc_data)
ct_data = X[cat_title]
X = sc_data.combine_first(ct_data).values
Y = Y.values
model.fit(X, Y, epochs=epochs, batch_size=batch_size, verbose=1)
save_model(dataset_name, model, target, scaler, non_cat_title,
cat_title)
def get_validation_function(args, model):
class ValidationFunction:
def __init__(self, func, iterations_per_epoch):
self.func = func
self.validation_iterations_per_epoch = iterations_per_epoch
if args.validation_mode == "none":
return None
if args.use_popdist and args.popdist_rank != 0:
# Run validation in a single process.
return None
if args.mixup_enabled or args.cutmix_enabled:
assert isinstance(model, augmentations.AugmentationModel)
model = model.model
opts = create_validation_opts(args, use_popdist=args.use_popdist)
test_data = datasets.get_data(args,
opts,
train=False,
async_dataloader=True,
return_remaining=True)
inference_model = poptorch.inferenceModel(model, opts)
def validation_func():
model.eval()
if inference_model._executable:
inference_model.attachToDevice()
val_acc = test(inference_model, test_data)
inference_model.detachFromDevice()
model.train()
return val_acc
return ValidationFunction(validation_func, len(test_data))
def get_features_types():
X, y = get_data('../data/trainDF.csv')
dtypes = pd.DataFrame(X.dtypes.rename('type')).reset_index().astype('str')
numeric = dtypes[(dtypes.type.isin(['int64', 'float64']))]['index'].values
categorical = dtypes[~(dtypes['index'].isin(numeric))
& (dtypes['index'] != 'y')]['index'].values
return categorical, numeric
def run_GB_learning(dataset_name, n_trees, learning_rate, max_depth):
for target in targets:
_, X, Y = dt.get_data(target=target, dataset_name=dataset_name)
# Модель
model = ensemble.GradientBoostingRegressor(n_estimators=n_trees,
learning_rate=learning_rate,
max_depth=max_depth)
# Обучение
if dataset_name == 'full harding':
non_cat_title = tl.full_hard_non_cat_title
cat_title = tl.full_hard_cat_title
elif dataset_name == 'double harding':
non_cat_title = tl.double_hard_non_cat_title
cat_title = tl.double_hard_cat_title
else:
raise ValueError
sc_data = X[non_cat_title]
scaler = preprocessing.StandardScaler()
sc_data = scaler.fit_transform(sc_data)
sc_data = pd.DataFrame(sc_data)
ct_data = X[cat_title]
X = sc_data.combine_first(ct_data).values
Y = Y.values
model.fit(X, Y)
save_model(dataset_name, model, target, scaler, non_cat_title,
cat_title)
def validate_checkpoints(checkpoint_list, test_data=None):
checkpoint = torch.load(checkpoint_list[0])
opts = checkpoint['opts']
utils.Logger.setup_logging_folder(opts)
# make sure the order is ascending
def ckpt_key(ckpt):
return int(ckpt.split('_')[-1].split('.')[0])
try:
checkpoint_list = sorted(checkpoint_list, key=ckpt_key)
except:
logging.warn("Checkpoint names are changed, which may cause inconsistent order in evaluation.")
model_opts = create_validation_opts(opts)
if test_data is None:
logging.info("Loading the data")
test_data = datasets.get_data(opts, model_opts, train=False, async_dataloader=True)
logging.info("Create model")
model = models.get_model(opts, datasets.datasets_info[opts.data], pretrained=False)
model.eval()
inference_model = poptorch.inferenceModel(model, model_opts)
for checkpoint in checkpoint_list:
load_checkpoint_weights(inference_model, checkpoint)
acc = test(inference_model, test_data, opts)
epoch_nr = torch.load(checkpoint)["epoch"]
result_dict = {"validation_epoch": epoch_nr,
"validation_iteration": opts.logs_per_epoch * epoch_nr,
"validation_accuracy": acc}
utils.Logger.log_validate_results(result_dict)
def lid_trend_through_training(model_name='ce',
dataset='mnist',
noise_type='sym',
noise_ratio=0.):
"""
plot the lid trend for clean vs noisy samples through training.
This can provide some information about manifold learning dynamics through training.
"""
print('Dataset: %s, noise type: %s, noise ratio: %.1f' %
(dataset, noise_type, noise_ratio))
lids, acc_train, acc_test = None, None, None
# get LID of raw inputs
lid_subset = 128
k = 20
X_train, Y_train, X_test, Y_test = get_data(dataset)
rand_idxes = np.random.choice(X_train.shape[0],
lid_subset * 10,
replace=False)
X_train = X_train[rand_idxes]
X_train = X_train.reshape((X_train.shape[0], -1))
lid_tmp = []
for i in range(10):
s = i * 128
e = (i + 1) * 128
lid_tmp.extend(mle_batch(X_train[s:e], X_train[s:e], k=k))
lid_X = np.mean(lid_tmp)
print('LID of input X: ', lid_X)
# load pre-saved to avoid recomputing
lid_saved = "log/lid_%s_%s_%s%s.npy" % (model_name, dataset, noise_type,
noise_ratio)
acc_saved = "log/acc_%s_%s_%s%s.npy" % (model_name, dataset, noise_type,
noise_ratio)
if os.path.isfile(lid_saved):
lids = np.load(lid_saved)
lids = np.insert(lids, 0, lid_X)
print(lids)
if os.path.isfile(acc_saved):
data = np.load(acc_saved)
acc_train = data[0][:]
acc_test = data[1][:]
acc_train = np.insert(acc_train, 0, 0.)
acc_test = np.insert(acc_test, 0, 0.)
plot(model_name, dataset, noise_ratio, lids, acc_train, acc_test)
def run_model(dataset_name,
model_name,
model,
plot,
target,
n_splits,
is_plot=False):
ns, x, y = dt.get_data(target=target, dataset_name=dataset_name)
# Обучение
kfold = StratifiedKFold(n_splits=n_splits, shuffle=True)
cvscores = []
cat_title, non_cat_title = get_titles_lists(dataset_name)
it = 0
for train, test in kfold.split(x, y):
scaler = preprocessing.StandardScaler()
x_train, y_train = get_x_y_data(x, y, train, scaler, non_cat_title,
cat_title)
x_test, y_test = get_x_y_data(x, y, test, scaler, non_cat_title,
cat_title)
model.fit(x_train.values, y_train)
# Тестирование
predicted = model.predict(x_test.values)
scores = metrics.mean_squared_error(y_test, predicted)
print('Error: {:.2f}'.format(scores))
cvscores.append(scores)
print('Corr: {:.2f}\n'.format(np.corrcoef(y_test, predicted)[0][1]))
save_histogram(predicted, y_test, it)
x_test_sc = pd.DataFrame(scaler.inverse_transform(
x_test[non_cat_title]),
columns=non_cat_title,
index=x_test.index)
x_test_sc[target] = y_test
x_test_sc[target + u'(сеть)'] = predicted
x_test_sc.combine_first(
pd.DataFrame(ns)).dropna().to_excel(dp.results[model_name] +
'/res{}.xlsx'.format(str(it)))
it += 1
if is_plot:
plot(model, x_test, y_test)
print("%.2f (+/- %.2f)" % (np.mean(cvscores), np.std(cvscores)))
return [np.mean(cvscores), np.std(cvscores)]
def validate_checkpoints(checkpoint_list, test_data=None):
checkpoint = torch.load(checkpoint_list[0])
args = checkpoint['args']
utils.Logger.setup_logging_folder(args)
# make sure the order is ascending
def ckpt_key(ckpt):
return int(ckpt.split('_')[-1].split('.')[0])
try:
checkpoint_list = sorted(checkpoint_list, key=ckpt_key)
except:
logging.warn(
"Checkpoint names are changed, which may cause inconsistent order in evaluation."
)
# Validate in a single instance
opts = create_validation_opts(args, use_popdist=False)
args.use_popdist = False
args.popdist_size = 1
if test_data is None:
test_data = datasets.get_data(args,
opts,
train=False,
async_dataloader=True,
return_remaining=True)
model = models.get_model(args,
datasets.datasets_info[args.data],
pretrained=False)
model.eval()
# Load the weights of the first checkpoint for the model
models.load_model_state_dict(model, checkpoint['model_state_dict'])
inference_model = poptorch.inferenceModel(model, opts)
for checkpoint in checkpoint_list:
if inference_model.isCompiled():
load_checkpoint_weights(inference_model, checkpoint)
val_accuracy = test(inference_model, test_data)
epoch_nr = torch.load(checkpoint)["epoch"]
log_data = {
"validation_epoch": epoch_nr,
"validation_iteration": epoch_nr * len(test_data),
"validation_accuracy": val_accuracy,
}
utils.Logger.log_validate_results(log_data)
def get_validation_function(opts, model):
if run_opts.validation_mode == "none":
return None
inference_model_opts = create_validation_opts(opts)
test_data = datasets.get_data(opts,
inference_model_opts,
train=False,
async_dataloader=True)
inference_model = poptorch.inferenceModel(model, inference_model_opts)
def validation_func():
model.eval()
if inference_model._executable:
inference_model.attachToDevice()
val_acc = test(inference_model, test_data, opts)
inference_model.detachFromDevice()
return val_acc
return validation_func
def run_nn(dataset_name,
target,
n_splits,
is_plot=False,
batch_size=batch_size,
epochs=epochs):
# Данные
ns, X, Y = dt.get_data(target=target, dataset_name=dataset_name)
# Обучение
kfold = StratifiedKFold(n_splits=n_splits, shuffle=True)
cvscores = []
non_cat_title, cat_title = get_titles_lists(dataset_name)
it = 0
for train, test in kfold.split(X, Y):
scaler = preprocessing.StandardScaler()
x_train, y_train = get_x_y_data(X, Y, train, scaler, non_cat_title,
cat_title)
x_test, y_test = get_x_y_data(X, Y, test, scaler, non_cat_title,
cat_title)
# Модель
model = md.get_simple_nn(X.shape[1])
history = model.fit(x_train.values,
y_train,
epochs=epochs,
batch_size=batch_size,
verbose=0)
# if is_plot:
# plot(history)
scores = model.evaluate(x_test.values,
y_test,
verbose=0,
batch_size=batch_size)
print("%s: %.2f" % (model.metrics_names[1], scores[1]))
cvscores.append(scores[1])
predicted = model.predict(x_test.values, batch_size=1)
print("corr: {:.2f}".format(
np.corrcoef(y_test, [i[0] for i in predicted])[0][1]))
save_histogram(predicted, y_test, it)
x_test_sc = pd.DataFrame(scaler.inverse_transform(
x_test[non_cat_title]),
columns=non_cat_title,
index=x_test.index)
x_test_sc[target] = y_test
x_test_sc[target + u'(сеть)'] = predicted
x_test_sc.combine_first(
pd.DataFrame(ns)).dropna().to_excel(dp.results['NN'] +
'/res{}.xlsx'.format(str(it)))
it += 1
print("%.2f (+/- %.2f)" % (np.mean(cvscores), np.std(cvscores)))
return [np.mean(cvscores), np.std(cvscores)]
import datasets
import math
from wrappers.correlation import ace
import numpy as np
from collections import defaultdict
sc = defaultdict(int)
total = defaultdict(int)
for ds in datasets.get_datasets():
print ds['name']
Xc, yc = datasets.get_data(ds, convert='numbers', standardize=False)
cat_cor = ace(Xc, yc, range(Xc.shape[1]), ds['rtype'])
num_cor = ace(Xc, yc, [], ds['rtype'])
for b in (0.4,0.5,0.6,0.7,0.8,0.85,0.9,0.95,1.0,1.1,1.2,1.3):
for i,c in enumerate(datasets.get_columns(ds)):
bias = len(np.unique(Xc[:,i]))*1.0/(b*Xc.shape[0])
guess = 'num' if num_cor[i]>cat_cor[i]-bias else 'cat'
if guess == 'cat' and i in datasets.get_column_index(ds) or guess == 'num' and i not in datasets.get_column_index(ds):
sc[b] += 1
total[b] += 1
#print '%-30s %10.7f %10.7f %10.7f %s' % (c,num_cor[i],cat_cor[i],bias,guess)
print '%6.2f %6.2f%%' % (b,sc[b]*100/total[b])
### Some constants ###
data_file = './data/stock.csv'
sequence_len = 12
test_size = 0.3
epochs = 15
learning_rate = 5e-2
ss = MinMaxScaler(feature_range=(-1, 1))
model = Network(rnn_cell='lstm', sigmoid=False, rnn_units=2)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
criterion = nn.MSELoss()
cosine_annealing = optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=learning_rate)
X_train, X_test = get_data(data_file,
sequence_len=sequence_len,
test_size=test_size)
X_train = ss.fit_transform(X_train)
X_test = ss.transform(X_test)
X_train = torch.Tensor(X_train).type(torch.FloatTensor)
X_test = torch.Tensor(X_test).type(torch.FloatTensor)
def _make_sequence_prediction(model, inputs):
outputs = []
print('[INFO] Predicting ... ')
with tqdm(total=len(inputs), file=sys.stdout, colour='green') as pbar:
for i, input_ in enumerate(inputs):
if (i == 0):
import datasets
dataset="pugeault"
(x_train, y_train), (x_test, y_test), input_shape,num_classes,labels= datasets.get_data(dataset)
print(f"Images shape {input_shape}")
print(f"classes {num_classes}, labels:\n {labels}")
print(f"Train samples: {y_train.shape[0]}, Test samples: {y_test.shape[0]}")
import matplotlib.pyplot as plt
print(x_train.shape)
initial_sample=0
samples=64
skip= y_train.shape[0] // samples
grid_cols=8
grid_rows=samples // grid_cols
if samples % grid_cols >0:
grid_rows+=1
f,axes=plt.subplots(grid_rows,grid_cols)
for axr in axes:
for ax in axr:
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
for i in range(samples):
i_sample=i*skip+initial_sample
klass = y_train[i_sample].argmax()
row=i // grid_cols
col=i % grid_cols
def advs_train(dataset='cifar-10',
loss_name='ce',
epochs=120,
dynamic_epoch=100,
batch_size=128,
fosc_max=0.5,
epsilon=0.031):
"""
Adversarial training with PGD attack.
"""
print(
'DynamicAdvsTrain - Data set: %s, loss: %s, epochs: %s, dynamic_epoch: %s, batch: %s, epsilon: %s'
% (dataset, loss_name, epochs, dynamic_epoch, batch_size, epsilon))
X_train, Y_train, X_test, Y_test = get_data(dataset,
clip_min=0.,
clip_max=1.,
onehot=True)
n_images = X_train.shape[0]
image_shape = X_train.shape[1:]
n_class = Y_train.shape[1]
print("n_images:", n_images, "n_class:", n_class, "image_shape:",
image_shape)
model = get_model(dataset,
input_shape=image_shape,
n_class=n_class,
softmax=True)
# model.summary()
# create loss
if loss_name == 'ce':
loss = cross_entropy
else:
print("New loss function should be defined first.")
return
optimizer = SGD(lr=0.01, decay=1e-4, momentum=0.9)
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
# data augmentation
if dataset in ['mnist']:
datagen = ImageDataGenerator()
elif dataset in ['cifar-10']:
datagen = ImageDataGenerator(rotation_range=10,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
else:
datagen = ImageDataGenerator(width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
datagen.fit(X_train)
# pgd attack for training
attack = LinfPGDAttack(model,
epsilon=epsilon,
eps_iter=epsilon / 4,
nb_iter=10,
random_start=True,
loss_func='xent',
clip_min=np.min(X_train),
clip_max=np.max(X_train))
# initialize logger
mylogger = Logger(K.get_session(),
model,
X_train,
Y_train,
X_test,
Y_test,
dataset,
loss_name,
epochs,
suffix='%s' % epsilon)
batch_iterator = datagen.flow(X_train, Y_train, batch_size=batch_size)
start_time = time.time()
for ep in range(epochs):
# learning rate decay
if (ep + 1) == 60:
lr = float(K.get_value(model.optimizer.lr))
K.set_value(model.optimizer.lr, lr / 10.0)
if (ep + 1) == 100:
lr = float(K.get_value(model.optimizer.lr))
K.set_value(model.optimizer.lr, lr / 10.0)
lr = float(K.get_value(model.optimizer.lr))
# a simple linear decreasing of fosc
fosc = fosc_max - fosc_max * (ep * 1.0 / dynamic_epoch)
fosc = np.max([fosc, 0.0])
steps_per_epoch = int(X_train.shape[0] / batch_size)
pbar = tqdm(range(steps_per_epoch))
for it in pbar:
batch_x, batch_y = batch_iterator.next()
batch_advs, fosc_batch = attack.perturb(K.get_session(), batch_x,
batch_y, batch_size, ep,
fosc)
probs = model.predict(batch_advs)
loss_weight = np.max(-batch_y * np.log(probs + 1e-12), axis=1)
if it == 0:
fosc_all = fosc_batch
else:
fosc_all = np.concatenate((fosc_all, fosc_batch), axis=0)
if ep == 0:
loss, acc = model.train_on_batch(batch_advs, batch_y)
else:
loss, acc = model.train_on_batch(batch_advs,
batch_y,
sample_weight=loss_weight)
pbar.set_postfix(acc='%.4f' % acc, loss='%.4f' % loss)
print('All time:', time.time() - start_time)
log_path = './log'
file_name = os.path.join(
log_path, 'BatchSize_{}_Epoch_{}_fosc.npy'.format(batch_size, ep))
np.save(file_name, fosc_all)
val_loss, val_acc = model.evaluate(X_test,
Y_test,
batch_size=batch_size,
verbose=0)
logs = {
'acc': acc,
'loss': loss,
'val_acc': val_acc,
'val_loss': val_loss
}
print(
"Epoch %s - loss: %.4f - acc: %.4f - val_loss: %.4f - val_acc: %.4f"
% (ep, loss, acc, val_loss, val_acc))
# save the log and model every epoch
mylogger.on_epoch_end(epoch=ep, logs=logs)
model.save_weights("model/advs_%s_%s_%s_%s.hdf5" %
(dataset, loss_name, epsilon, ep))
def main():
dataset = 'mnist'
batch_size = 100
n_classes = 10
learning_rate = 1e-3
name = dataset
oldmodel = None
inits = {
'weights_init': IsotropicGaussian(0.01),
'biases_init': Constant(0.),
}
# -----------------------------------------------------------------------
image_size, channels, data_train, data_valid, data_test = \
datasets.get_data(dataset)
img_height, img_width = image_size
train_stream = Flatten(DataStream.default_stream(
data_train, iteration_scheme=SequentialScheme(
data_train.num_examples, batch_size)))
test_stream = Flatten(DataStream.default_stream(
data_test, iteration_scheme=SequentialScheme(
data_test.num_examples, batch_size)))
logging.info("experiment dataset: %s" % dataset)
# --------------- Building Model ----------------
glim_net = GlimpseNetwork(dim=128,
n_channels=channels, img_height=img_height,
img_width=img_width, N=7, name='glimpse_net',
**inits) # output (n)
core = CoreNetwork(input_dim=128, dim=256, name='core_net',
**inits)
loc_net = LocationNetwork(input_dim=256, loc_emb=2, std=0.18,
non_hetro=False, name='loc_net', **inits)
action = ActionNetwork(input_dim=256, n_classes=n_classes,
name='action', **inits)
ram = RAM(core=core, glimpes_network=glim_net,
location_network=loc_net, action_network=action,
n_steps=8, name='RAM', random_init_loc=False, **inits)
ram.initialize()
# -------------------------------------------------------------
img = T.matrix('features')
y = T.imatrix('targets')
y = y.flatten()
loc_sample, actions, _, _, loc_u = ram.out(img)
loc_sample = loc_sample[:-1]
loc_u = loc_u[:-1]
# classification task
actions = T.repeat(actions[1:][-1][None, :, :], loc_sample.shape[0], axis=0)
# get loc network param
# ---------------- Building Reinforce alforithm ----------------
loc_bricks = list(loc_net.children) # + list(glim_net.children)
others_bricks = list(core.children) + list(action.children) + \
list(glim_net.children)
reinforce = REINFORCE()
cost_re, reward, baseline = \
reinforce.build_reinforce_cost_reward_base(y, actions)
cg_rein = ComputationGraph([cost_re])
loc_params = VariableFilter(roles=[PARAMETER],
bricks=loc_bricks)(cg_rein.variables)
loc_grad = reinforce.build_reinforce_grad(cost_re, loc_u,
loc_sample, loc_params, reward,
baseline, loc_net.std)
y_dis = actions[-1]
cost_true = CategoricalCrossEntropy().apply(y, y_dis)
# cost_true = cost_re
cg = ComputationGraph([cost_true])
# filter out initial_state
others_params = VariableFilter(roles=[PARAMETER],
bricks=others_bricks)(cg.variables)
other_grad = T.grad(cost_true, others_params)
# Hybrid Cost
all_grad = loc_grad + other_grad
all_params = loc_params + others_params
gradients = {param: grad for param, grad in zip(all_params, all_grad)}
algorithm = GradientDescent(
cost=cost_true,
gradients=gradients,
parameters=all_params,
step_rule=CompositeRule([
StepClipping(10.),
Adam(learning_rate)]),
)
algorithm.add_updates(cg_rein.updates)
# ------------------------------------------------------------------------
# Setup monitors
cost_true.name = 'cost_true'
cost_re.name = 'reinforce cost'
# avg_action = actions.mean(1)
# avg_action.name = 'avg_action'
acc = T.cast(T.eq(y, T.argmax(y_dis, axis=1)), 'float32').mean()
acc.name = 'accuratcy'
monitors = [cost_re, acc, cost_true]
train_monitors = monitors
monitor_img = data_train.data_sources[0][:10*20].reshape((200, -1))
# Live plotting...
# plot_channels = [
# ["cost_true", "reinforce_cost"]
# ]
# ------------------------------------------------------------
# plotting_extensions = [
# Plot(name, channels=plot_channels)
# ]
subdir = './exp/' + name + "-" + time.strftime("%Y%m%d-%H%M%S")
if not os.path.exists(subdir):
os.makedirs(subdir)
main_loop = MainLoop(
model=Model(cost_true),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
EarlyStopping('test_cost_true', iterations=100),
FinishAfter(after_n_epochs=500),
TrainingDataMonitoring(
train_monitors,
prefix="train",
after_epoch=True),
DataStreamMonitoring(
monitors,
test_stream,
updates=cg_rein.updates,
prefix="test"),
ProgressBar(),
Printing(),
SampleAttentionCheckpoint(monitor_img=monitor_img,
image_size=image_size[0],
channels=channels,
save_subdir='{}'.format(subdir),
before_training=True,
after_epoch=True),
PartsOnlyCheckpoint("{}/{}".format(subdir, name),
before_training=True, after_epoch=True,
save_separately=['log', 'model'])])
if oldmodel is not None:
print("Initializing parameters with old model %s" % oldmodel)
with open(oldmodel, "rb") as f:
oldmodel = pickle.load(f)
main_loop.model.set_parameter_values(
oldmodel.get_parameter_values())
del oldmodel
main_loop.run()
import sklearn.model_selection
import sklearn.metrics
import pandas as pd
import numpy as np
# Import files from source directory
import os
import sys
module_path = os.path.abspath(os.path.join('../src/'))
if module_path not in sys.path:
sys.path.append(module_path)
import hyperplane, datasets
from model import BATDepth
PROCESSED_DATASETS = {
# 'wpbc': datasets.get_data("../data/breast-cancer/wpbc.data", 'wpbc'),
# 'wdbc': datasets.get_data("../data/breast-cancer/wdbc.data", 'wdbc'),
# 'breast_cancer_yugoslavia': datasets.get_data("../data/breast-cancer/breast-cancer.data", 'breast_cancer_yugoslavia'),
'breast_cancer_wisconsin':
datasets.get_data("../data/breast-cancer/breast-cancer-wisconsin.data",
'breast_cancer_wisconsin'),
}
model, train_data, test_data = datasets.get_model(
PROCESSED_DATASETS['breast_cancer_wisconsin'])
c_tree = BATDepth(model, log=False)
output_decision_tree = c_tree.fit()
test_predict = output_decision_tree.predict(
test_data[test_data.columns.difference(['target'])].values)
print('BATDepth accuracy: {}'.format(
(test_predict == test_data['target']).sum() / len(test_predict)))
import numpy as np
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
import datasets
from sklearn.cross_validation import cross_val_score, KFold
from sklearn.metrics import make_scorer, mean_absolute_error
from wrappers.correlation import ace,rcorrs,rcorrp, bcdcor
from collections import defaultdict
MAD = make_scorer(mean_absolute_error, greater_is_better=False)
sc=defaultdict(lambda:defaultdict(lambda:defaultdict(int)))
for ds in datasets.get_datasets():
Xc, yc = datasets.get_data(ds, convert='numbers', standardize=False)
kf = KFold(Xc.shape[0], 5, shuffle=True)
for vi in (bcdcor, rcorrp, rcorrs, ace):
cor = vi(Xc, yc, datasets.get_column_index(ds))
#print zip(ds['category']+ds['numeric'],cor)
for drop in range(10):
vi_name = vi.__name__
X, y = datasets.get_data(ds, drop=np.argsort(cor)[:drop])
if ds['rtype'] == 'Binary':
metric = 'log_loss'
clf = RandomForestClassifier(n_estimators=200,min_samples_leaf=5,n_jobs=1)
else:
metric = MAD
clf = RandomForestRegressor(n_estimators=200,min_samples_leaf=5,n_jobs=1)
sc[ds['name']][vi_name][drop] = cross_val_score(clf, X, y, cv=kf, n_jobs=3, scoring=metric).mean()
print ds['name']+' '+vi_name+" "+str(drop)+' '+str(sc[ds['name']][vi_name][drop])
if len(sys.argv) > 1 and sys.argv[1] == 'r':
resume = True
hidden_number = 256
learning_rate = 0.01
bs = 128
saving_path = 'models'
start_epoch = 0
end_epoch = 200
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
cuda_available = torch.cuda.is_available()
#cuda_available = False
para_padding_a, word2ix, ix2word = ds.get_data()
para_padding_t = torch.from_numpy(para_padding_a)
net = nets.PoetryModel(hidden_number, len(word2ix))
if resume is True:
checkpoints = torch.load(os.path.join(saving_path, 'best_model.t7'))
net.load_state_dict(checkpoints['net'])
start_epoch = checkpoints['epoch']
net = net.cuda() if cuda_available is True else net
dataloader = DataLoader(para_padding_t, batch_size = bs, shuffle = True, num_workers = 1)
optimizer = torch.optim.Adam(net.parameters(), lr = learning_rate)
criterion = nn.CrossEntropyLoss()
if opts.warmup_epoch > 0:
lr_scheduler = WarmUpLRDecorator(optimizer=optimizer,
lr_scheduler=lr_scheduler,
warmup_epoch=opts.warmup_epoch)
return lr_scheduler
if __name__ == '__main__':
run_opts = parse_arguments()
logging.info("Loading the data")
model_opts = create_model_opts(run_opts)
train_data = datasets.get_data(run_opts,
model_opts,
train=True,
async_dataloader=True)
logging.info("Initialize the model")
model = models.get_model(run_opts,
datasets.datasets_info[run_opts.data],
pretrained=False)
model.train()
optimizer = get_optimizer(run_opts, model)
lr_scheduler = get_lr_scheduler(run_opts, optimizer)
training_model = convert_to_ipu_model(model, run_opts, optimizer)
training_validation_func = get_validation_function(
run_opts, model) if run_opts.validation_mode == "during" else None
train(training_model, train_data, run_opts, lr_scheduler,
range(1, run_opts.epoch + 1), optimizer, training_validation_func)
if act[i]==0:
sc += 2 * pred[i]
else:
if pred[i]<=0:
sc += 2 * (act[i] * math.log(act[i]/0.1) - (act[i] - pred[i]))
else:
sc += 2 * (act[i] * math.log(act[i]/pred[i]) - (act[i] - pred[i]))
return sc/act.shape[0]
RMSLE = make_scorer(rmsle, greater_is_better=False)
PSDEV = make_scorer(psdev, greater_is_better=False)
#for ds in datasets.get_datasets(rtype='Positive'): #,name='census_1990_small'):
for ds in datasets.get_datasets(name='census_2012h_small'):
#if 'census' in ds['name']: continue
X, y = datasets.get_data(ds,standardize=False)
kf = KFold(X.shape[0], 2, shuffle=True, random_state=1234)
for a in (0.1, 0.3, 0.6, 0.9):
for lm in (1, 0.1, 0.01):
clf1 = CGLM(distribution='Poisson',trace=False)
clf2 = Ridge(alpha=lm)
clf3 = ElasticNetC(distribution='Poisson', alpha=lm, l1_ratio=a, tolerance=0.001)
#clf3t = ElasticNetC(distribution='Tweedie', alpha=lm, l1_ratio=a, tolerance=0.001,p=1.5)
#clf3g = ElasticNetC(distribution='Gamma', alpha=lm, l1_ratio=a, tolerance=0.001)
clf4 = GlmnetWrapper(**{'family': 'gaussian', 'alpha': a, 'lambda': lm, 'maxit': 300})
#clf4 = GlmnetWrapper(**{'family': 'poisson', 'alpha': a, 'lambda': lm, 'maxit': 300})
score_func = PSDEV #'roc_auc'
for clf in (clf1, clf2, clf3, clf4):
#for clf in (clf1, clf2, clf3, clf3t, clf3g, clf4):
if clf != clf1 or (a == 0.1 and lm == 10):
cn=clf.__class__.__name__
import sklearn.ensemble
import sklearn.model_selection
import sklearn.metrics
import pandas as pd
import numpy as np
# Import files from source directory
import os
import sys
module_path = os.path.abspath(os.path.join('../src/'))
if module_path not in sys.path:
sys.path.append(module_path)
import hyperplane, datasets
from model import BATDepth
PROCESSED_DATASETS = {
'htru': datasets.get_data("../data/htru/HTRU_2.csv", 'htru'),
}
model, train_data, test_data = datasets.get_model(PROCESSED_DATASETS['htru'], n_estimators = 2)
c_tree = BATDepth(model, log = False)
output_decision_tree = c_tree.fit()
test_predict = output_decision_tree.predict(test_data[test_data.columns.difference(['target'])].values)
print('BATDepth accuracy: {}'.format((test_predict == test_data['target']).sum()/len(test_predict)))
y.backward(torch.ones_like(y))
jacob = x.grad.detach()
return jacob, target.detach(), y.detach(), out.detach()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
savedataset = args.dataset
dataset = 'fake' if 'fake' in args.dataset else args.dataset
args.dataset = args.dataset.replace('fake', '')
if args.dataset == 'cifar10':
args.dataset = args.dataset + '-valid'
searchspace = nasspace.get_search_space(args)
if 'valid' in args.dataset:
args.dataset = args.dataset.replace('-valid', '')
train_loader = datasets.get_data(args.dataset, args.data_loc, args.trainval,
args.batch_size, args.augtype, args.repeat,
args)
os.makedirs(args.save_loc, exist_ok=True)
filename = f'{args.save_loc}/{args.save_string}_{args.score}_{args.nasspace}_{savedataset}{"_" + args.init + "_" if args.init != "" else args.init}_{"_dropout" if args.dropout else ""}_{args.augtype}_{args.sigma}_{args.repeat}_{args.trainval}_{args.batch_size}_{args.maxofn}_{args.seed}'
accfilename = f'{args.save_loc}/{args.save_string}_accs_{args.nasspace}_{savedataset}_{args.trainval}'
if args.dataset == 'cifar10':
acc_type = 'ori-test'
val_acc_type = 'x-valid'
else:
acc_type = 'x-test'
val_acc_type = 'x-valid'
scores = np.zeros(len(searchspace))
try:
import data_paths as dp
import titles as tl
import datasets as dt
from matplotlib import pyplot as plt
from sklearn.model_selection import StratifiedKFold
from sklearn import preprocessing
from sklearn import svm
from sklearn import neighbors
# Параметры обучения
C = 10.0
n_trees = 256
# Данные
target = u'прочность'
ns, X, Y = dt.get_data(target=target)
# Обучение
kfold = StratifiedKFold(n_splits=10, shuffle=True)
cvscores = []
it = 0
for train, test in kfold.split(X, Y):
scaler = preprocessing.StandardScaler()
sc_data = X.loc[train, tl.full_hard_non_cat_title]
sc_data = pd.DataFrame(scaler.fit_transform(sc_data),
index=sc_data.index,
columns=sc_data.columns)
ct_data = X.loc[train, tl.full_hard_cat_title]
x_train = pd.concat([sc_data, ct_data], axis=1)
def get_numeric_features():
X, y = get_data('../data/trainDF.csv')
dtypes = pd.DataFrame(X.dtypes.rename('type')).reset_index().astype('str')
numeric = dtypes[(dtypes.type.isin(['int64', 'float64']))]['index'].values
return numeric
)
args.normalization_location = 'ipu'
utils.handle_distributed_settings(args)
return args
if __name__ == '__main__':
args = parse_arguments()
utils.Logger.setup_logging_folder(args)
if args.use_popdist:
opts = popdist.poptorch.Options(
ipus_per_replica=len(args.pipeline_splits) + 1)
else:
opts = poptorch.Options()
opts.replicationFactor(args.replicas)
opts.deviceIterations(args.device_iterations)
dataloader = datasets.get_data(args,
opts,
train=False,
async_dataloader=False)
model = models.get_model(args,
datasets.datasets_info[args.data],
pretrained=not args.random_weights)
model.eval()
opts = utils.inference_settings(args, copy.deepcopy(opts))
inference_model = poptorch.inferenceModel(model, opts)
benchmark(inference_model, dataloader, args)
from sklearn.preprocessing import StandardScaler
import pickle
import os
model_name = 'OneClassSVM'
#nu = 0.0005
kernel = 'rbf'
#gamma = 0.0002
#training path
model_home = os.getcwd()
normal_path = model_home + '/datasets/normal2.xlsx'
abnormal_path = model_home + '/datasets/abnormal.xlsx'
print(abnormal_path)
X_train, X_val, val_set, test_set = datasets.get_data(model_name, normal_path,
abnormal_path)
sc = StandardScaler()
sc.fit(X_train)
X_train_std = sc.transform(X_train)
def fn(args):
nu, gamma = args
params = {'nu': nu, 'kernel': kernel, 'gamma': gamma}
oneclasssvm = creat_model(model_name, params)
oneclasssvm.fit(X_train_std)
errors = 0
for data in val_set:
X = data.X
y = data.y
help='learning rate of optimizer')
parser.add_argument('--seed', default=45, type=int)
# LTEC hyperparameters
parser.add_argument('-m', '--num_of_prediction', default=5, type=int)
parser.add_argument('-w', '--warming_up_period', default=10, type=int)
config = parser.parse_args()
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = '%d' % config.gpu
# load dataset
train_x, train_y, test_x, test_y, clean_y = get_data(
dataset=config.dataset,
noise_ratio=config.noise_ratio,
noise_type=config.noise_type,
random_shuffle=True,
seed=config.seed)
np.random.seed(config.seed)
n_samples = train_x.shape[0]
x_shape = train_x.shape[1:]
y_shape = train_y.shape[1]
n_batches = n_samples // batch_size + 1
# define model
model = convnet(config, x_shape=x_shape, y_shape=y_shape, mom2=0.1)
forget_rate = config.noise_ratio / 100.
remember_rate = 1. - forget_rate
import datasets
from wrappers.elastic_net_c import ElasticNetC
import time
from wrappers.glmnet import GlmnetWrapper
from wrappers.zeroinfl import ZeroInflWrapper
from wrappers.glm_c import CGLM
from sklearn.cross_validation import cross_val_score, KFold
from sklearn.metrics import make_scorer, mean_absolute_error
from sklearn.linear_model import SGDRegressor, Ridge, LogisticRegression, SGDClassifier, ElasticNet
MAD = make_scorer(mean_absolute_error, greater_is_better=False)
score_func='r2'
for ds in datasets.get_datasets(name='mets_short'):
X, y = datasets.get_data(ds)
kf = KFold(X.shape[0], 15, shuffle=True, random_state=1234)
for a in (0.1, 0.5, 1.0, 1.5):
clf1 = Ridge(alpha=a)
clf2 = CGLM(distribution='Poisson',trace=False)
clf3 = ZeroInflWrapper()
for clf in (clf1,clf2,clf3):
if a!=0.1 and clf!=clf1:
continue
st = time.time()
print '%-20s %20s a %3.1f sc %9.4f tm %5.2f' % \
(ds['name'], clf.__class__.__name__, a,
cross_val_score(clf, X, y, cv=kf, scoring=score_func, n_jobs=3).mean(), time.time()-st)
def feature_visualization(model_name='ce',
dataset='mnist',
num_classes=10,
noise_ratio=40,
n_samples=100):
"""
This is to show how features of incorretly labeled images are overffited to the wrong class.
plot t-SNE 2D-projected deep features (right before logits).
This will generate 3 plots in a grid (3x1).
The first shows the raw features projections of two classes of images (clean label + noisy label)
The second shows the deep features learned by cross-entropy after training.
The third shows the deep features learned using a new loss after training.
:param model_name: a new model other than crossentropy(ce), can be: boot_hard, boot_soft, forward, backward, lid
:param dataset:
:param num_classes:
:param noise_type；
:param noise_ratio:
:param epochs: to find the last epoch
:param n_samples:
:return:
"""
print('Dataset: %s, model_name: ce/%s, noise ratio: %s%%' %
(model_name, dataset, noise_ratio))
features_ce = np.array([None, None])
features_other = np.array([None, None])
# # load pre-saved to avoid recomputing
# feature_tmp = "lof/representation_%s_%s.npy" % (dataset, noise_ratio)
# if os.path.isfile(feature_tmp):
# data = np.load(feature_tmp)
# features_input = data[0]
# features_ce = data[1]
# features_other = data[2]
#
# plot(model_name, dataset, noise_ratio, features_input, features_ce, features_other)
# return
# load data
X_train, Y_train, X_test, Y_test = get_data(dataset)
Y_noisy = np.load("data/noisy_label_%s_%s.npy" % (dataset, noise_ratio))
Y_noisy = Y_noisy.reshape(-1)
# sample training set
cls_a = 0
cls_b = 3
# find smaples labeled to class A and B
cls_a_idx = np.where(Y_noisy == cls_a)[0]
cls_b_idx = np.where(Y_noisy == cls_b)[0]
# sampling for efficiency purpose
cls_a_idx = np.random.choice(cls_a_idx, n_samples, replace=False)
cls_b_idx = np.random.choice(cls_b_idx, n_samples, replace=False)
X_a = X_train[cls_a_idx]
X_b = X_train[cls_b_idx]
image_shape = X_train.shape[1:]
model = get_model(dataset, input_tensor=None, input_shape=image_shape)
sgd = SGD(lr=0.01, momentum=0.9)
#### get deep representations of ce model
model_path = 'model/ce_%s_%s.hdf5' % (dataset, noise_ratio)
model.load_weights(model_path)
model.compile(loss=cross_entropy, optimizer=sgd, metrics=['accuracy'])
rep_a = get_deep_representations(model, X_a, batch_size=100).reshape(
(X_a.shape[0], -1))
rep_b = get_deep_representations(model, X_b, batch_size=100).reshape(
(X_b.shape[0], -1))
rep_a = TSNE(n_components=2).fit_transform(rep_a)
rep_b = TSNE(n_components=2).fit_transform(rep_b)
features_ce[0] = rep_a
features_ce[1] = rep_b
#### get deep representations of other model
model_path = 'model/%s_%s_%s.hdf5' % (model_name, dataset, noise_ratio)
model.load_weights(model_path)
model.compile(loss=cross_entropy, optimizer=sgd, metrics=['accuracy'])
rep_a = get_deep_representations(model, X_a, batch_size=100).reshape(
(X_a.shape[0], -1))
rep_b = get_deep_representations(model, X_b, batch_size=100).reshape(
(X_b.shape[0], -1))
rep_a = TSNE(n_components=2).fit_transform(rep_a)
rep_b = TSNE(n_components=2).fit_transform(rep_b)
features_other[0] = rep_a
features_other[1] = rep_b
# plot
fig = plt.figure(figsize=(12, 5))
gs = gridspec.GridSpec(1, 2, wspace=0.15)
a_clean_idx = Y_train[cls_a_idx] == Y_noisy[cls_a_idx]
a_noisy_idx = Y_train[cls_a_idx] != Y_noisy[cls_a_idx]
b_clean_idx = Y_train[cls_b_idx] == Y_noisy[cls_b_idx]
b_noisy_idx = Y_train[cls_b_idx] != Y_noisy[cls_b_idx]
## plot features learned by cross-entropy
ax = fig.add_subplot(gs[0, 0])
A = features_ce[0]
B = features_ce[1]
# clean labeld class A samples plot
ax.scatter(A[a_clean_idx][:, 0].ravel(),
A[a_clean_idx][:, 1].ravel(),
c='b',
marker='o',
s=10,
label='class A: clean')
ax.scatter(A[a_noisy_idx][:, 0].ravel(),
A[a_noisy_idx][:, 1].ravel(),
c='m',
marker='x',
s=30,
label='class A: noisy')
ax.scatter(B[b_clean_idx][:, 0].ravel(),
B[b_clean_idx][:, 1].ravel(),
c='r',
marker='o',
s=10,
label='class B: clean')
ax.scatter(B[b_noisy_idx][:, 0].ravel(),
B[b_noisy_idx][:, 1].ravel(),
c='c',
marker='x',
s=30,
label='class B: noisy')
ax.set_title("cross-entropy", fontsize=15)
legend = ax.legend(loc='lower center', ncol=2)
plt.setp(legend.get_texts(), fontsize=15)
ax = fig.add_subplot(gs[0, 1])
A = features_other[0]
B = features_other[1]
ax.scatter(A[a_clean_idx][:, 0].ravel(),
A[a_clean_idx][:, 1].ravel(),
c='b',
marker='o',
s=10,
label='class A: clean')
ax.scatter(A[a_noisy_idx][:, 0].ravel(),
A[a_noisy_idx][:, 1].ravel(),
c='m',
marker='x',
s=30,
label='class A: noisy')
ax.scatter(B[b_clean_idx][:, 0].ravel(),
B[b_clean_idx][:, 1].ravel() - 5,
c='r',
marker='o',
s=10,
label='class B: clean')
ax.scatter(B[b_noisy_idx][:, 0].ravel(),
B[b_noisy_idx][:, 1].ravel(),
c='c',
marker='x',
s=30,
label='class B: noisy')
ax.set_title("D2L", fontsize=15)
legend = ax.legend(loc='lower center', ncol=2)
plt.setp(legend.get_texts(), fontsize=15)
fig.savefig("plots/representations_%s_%s_%s.png" %
(model_name, dataset, noise_ratio),
dpi=300)
plt.show()
# epochs.append(d3 // 2)
# epochs.append(d3 // 2)
# curriculum_type = "hard"
score_path = dataset + "_scores.p"
os.environ["PYTHONHASHSEED"] = str(seed)
random.seed(seed)
np.random.seed(seed)
tf.random.set_seed(seed)
X_train, y_train, X_test, y_test = datasets.get_data(
dataset)
tf.keras.backend.clear_session()
gc.collect()
if dataset == 'bert_imdb1' or True:
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver(
)
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
strategy = tf.distribute.experimental.TPUStrategy(
tpu)
# Create model
with strategy.scope():
model = models.get_model(dataset)
import numpy as np
import datasets
def get_accuracy_metric(col):
eda_report = {}
rp = _get_report_level_one(eda_report, col, col.dtype)
return rp['metric_options']['accuracy']['short_name']
#for ds in datasets.get_datasets(name=['fastiron-train-30k','kickcars_train_full','census_1990_small','french_damage_cost','allstate_nonzero_small','amazon_small_no-c','credit_full','bank_marketing_small','bio_grid_small','mets','trainingDataWithoutNegativeWeights','census_2012h_small']):
#for ds in datasets.get_datasets(name='trainingDataWithoutNegativeWeights'):
for ds in datasets.get_datasets(name='fastiron-train-30k'):
#for ds in datasets.get_datasets(name='fastiron_small'):
#for ds in datasets.get_datasets(name='kickcars_train_full'):
#for ds in datasets.get_datasets(name='french_damage_cost'):
#for ds in datasets.get_datasets(name='census_1990_small'):
X, y = datasets.get_data(ds,standardize=False,convert='numbers')
cols = datasets.get_columns(ds)
cats = datasets.get_columns(ds,'category')
sc1={}
sc2={}
klist=(0,)
#klist = (1,5,10,20,40,80)
df=pandas.DataFrame(np.hstack((X,np.reshape(y,(-1,1)))),columns=cols+[ds['target']])
rm = get_accuracy_metric(df[ds['target']])
mdir = direction_by_name(rm)
mfunc = metric_by_name(rm)
rnk1={}
rnk2={}
for K in klist:
sys.stderr.write( ds['name']+' '+str(K)+'\n')
if ds['rtype']=='Binary':
def train(dataset='mnist', model_name='d2l', batch_size=128, epochs=50, noise_ratio=0):
"""
Train one model with data augmentation: random padding+cropping and horizontal flip
:param dataset:
:param model_name:
:param batch_size:
:param epochs:
:param noise_ratio:
:return:
"""
print('Dataset: %s, model: %s, batch: %s, epochs: %s, noise ratio: %s%%' %
(dataset, model_name, batch_size, epochs, noise_ratio))
# load data
X_train, y_train, X_test, y_test = get_data(dataset, noise_ratio, random_shuffle=True)
# X_train, y_train, X_val, y_val = validatation_split(X_train, y_train, split=0.1)
n_images = X_train.shape[0]
image_shape = X_train.shape[1:]
num_classes = y_train.shape[1]
print("n_images", n_images, "num_classes", num_classes, "image_shape:", image_shape)
# load model
model = get_model(dataset, input_tensor=None, input_shape=image_shape, num_classes=num_classes)
# model.summary()
optimizer = SGD(lr=0.01, decay=1e-4, momentum=0.9)
# for backward, forward loss
# suppose the model knows noise ratio
P = uniform_noise_model_P(num_classes, noise_ratio/100.)
# create loss
if model_name == 'forward':
P = uniform_noise_model_P(num_classes, noise_ratio / 100.)
loss = forward(P)
elif model_name == 'backward':
P = uniform_noise_model_P(num_classes, noise_ratio / 100.)
loss = backward(P)
elif model_name == 'boot_hard':
loss = boot_hard
elif model_name == 'boot_soft':
loss = boot_soft
elif model_name == 'd2l':
loss = lid_paced_loss()
else:
loss = cross_entropy
# model
model.compile(
loss=loss,
optimizer=optimizer,
metrics=['accuracy']
)
## do real-time updates using callbakcs
callbacks = []
if model_name == 'd2l':
init_epoch = D2L[dataset]['init_epoch']
epoch_win = D2L[dataset]['epoch_win']
d2l_learning = D2LCallback(model, X_train, y_train,
dataset, noise_ratio,
epochs=epochs,
pace_type=model_name,
init_epoch=init_epoch,
epoch_win=epoch_win)
callbacks.append(d2l_learning)
cp_callback = ModelCheckpoint("model/%s_%s_%s.hdf5" % (model_name, dataset, noise_ratio),
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True,
period=1)
callbacks.append(cp_callback)
else:
cp_callback = ModelCheckpoint("model/%s_%s_%s.hdf5" % (model_name, dataset, noise_ratio),
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True,
period=epochs)
callbacks.append(cp_callback)
# tensorboard callback
callbacks.append(TensorBoard(log_dir='./log/log'))
# learning rate scheduler if use sgd
lr_scheduler = get_lr_scheduler(dataset)
callbacks.append(lr_scheduler)
# acc, loss, lid
log_callback = LoggerCallback(model, X_train, y_train, X_test, y_test, dataset,
model_name, noise_ratio, epochs)
callbacks.append(log_callback)
# data augmentation
if dataset in ['mnist', 'svhn']:
datagen = ImageDataGenerator()
elif dataset in ['cifar-10', 'cifar-100']:
datagen = ImageDataGenerator(
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
else:
datagen = ImageDataGenerator(
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True)
datagen.fit(X_train)
# train model
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
steps_per_epoch=len(X_train) / batch_size, epochs=epochs,
validation_data=(X_test, y_test),
verbose=1,
callbacks=callbacks
)
""" Transform the mnist dataset into a bigger image, to test the effect of
attention.
"""
from fuel.datasets.hdf5 import H5PYDataset
import h5py
import numpy as np
import datasets
image_size, channels, data_train, data_valid, data_test = \
datasets.get_data('mnist')
data_sources = np.vstack([data_train.data_sources[0],
data_test.data_sources[0],
data_valid.data_sources[0]])
data_targets = np.vstack([data_train.data_sources[1],
data_test.data_sources[1],
data_valid.data_sources[1]])
batch_size = data_sources.shape[0]
data_buffer = np.zeros((batch_size, 1, 60, 60))
# create the index
start = np.random.random_integers(
low=0, high=60 - 28, size=(batch_size, 2)) + 28 // 2
img_idx = np.arange(60 * 60).reshape(60, 60)
for i, j in zip(range(batch_size), start):
x, y = j.tolist()
data_buffer[i, :, x-14: x+14, y-14: y+14] = \
def train(dataset='mnist',
model_name='sl',
batch_size=128,
epochs=50,
noise_ratio=0,
asym=False,
alpha=1.0,
beta=1.0):
"""
Train one model with data augmentation: random padding+cropping and horizontal flip
:param dataset:
:param model_name:
:param batch_size:
:param epochs:
:param noise_ratio:
:return:
"""
print(
'Dataset: %s, model: %s, batch: %s, epochs: %s, noise ratio: %s%%, asymmetric: %s, alpha: %s, beta: %s'
% (dataset, model_name, batch_size, epochs, noise_ratio, asym, alpha,
beta))
# load data
X_train, y_train, y_train_clean, X_test, y_test = get_data(
dataset, noise_ratio, asym=asym, random_shuffle=False)
n_images = X_train.shape[0]
image_shape = X_train.shape[1:]
num_classes = y_train.shape[1]
print("n_images", n_images, "num_classes", num_classes, "image_shape:",
image_shape)
# load model
model = get_model(dataset,
input_tensor=None,
input_shape=image_shape,
num_classes=num_classes)
# model.summary()
if dataset == 'cifar-100':
optimizer = SGD(lr=0.1, decay=5e-3, momentum=0.9)
else:
optimizer = SGD(lr=0.1, decay=1e-4, momentum=0.9)
# create loss
if model_name == 'ce':
loss = cross_entropy
elif model_name == 'sl':
loss = symmetric_cross_entropy(alpha, beta)
elif model_name == 'lsr':
loss = lsr
elif model_name == 'joint':
loss = joint_optimization_loss
elif model_name == 'gce':
loss = generalized_cross_entropy
elif model_name == 'boot_hard':
loss = boot_hard
elif model_name == 'boot_soft':
loss = boot_soft
elif model_name == 'forward':
loss = forward(P)
elif model_name == 'backward':
loss = backward(P)
else:
print("Model %s is unimplemented!" % model_name)
exit(0)
# model
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
if asym:
model_save_file = "model/asym_%s_%s_%s.{epoch:02d}.hdf5" % (
model_name, dataset, noise_ratio)
else:
model_save_file = "model/%s_%s_%s.{epoch:02d}.hdf5" % (
model_name, dataset, noise_ratio)
## do real-time updates using callbakcs
callbacks = []
if model_name == 'sl':
cp_callback = ModelCheckpoint(model_save_file,
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True,
period=1)
callbacks.append(cp_callback)
else:
cp_callback = ModelCheckpoint(model_save_file,
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=True,
period=1)
callbacks.append(cp_callback)
# learning rate scheduler if use sgd
lr_scheduler = get_lr_scheduler(dataset)
callbacks.append(lr_scheduler)
callbacks.append(SGDLearningRateTracker(model))
# acc, loss, lid
log_callback = LoggerCallback(model, X_train, y_train, y_train_clean,
X_test, y_test, dataset, model_name,
noise_ratio, asym, epochs, alpha, beta)
callbacks.append(log_callback)
# data augmentation
if dataset in ['mnist', 'svhn']:
datagen = ImageDataGenerator()
elif dataset in ['cifar-10']:
datagen = ImageDataGenerator(width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
else:
datagen = ImageDataGenerator(rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
datagen.fit(X_train)
# train model
model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
steps_per_epoch=len(X_train) / batch_size,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=1,
callbacks=callbacks)
def train(epoch=10, batch_size=128, log_img=True):
data, batch_count = [], 0
if dataset == 'cifar10' or dataset == 'mnist' or dataset == 'cifar100' or dataset == 'o-faces':
data, batch_count = datasets.get_data(
dataset, (input_width, input_height),
(output_width, output_height), color_channels, batch_size)
else:
if dataset == 'celebA':
data, batch_count = datasets.get_data(
dataset, (input_width, input_height),
(output_width, output_height), batch_size, color_channels)
elif dataset == 'letters':
data, batch_count = datasets.get_data(
dataset, (input_width, input_height),
(output_width, output_height),
batch_size,
color_channels,
file_ext='*.png')
for i in range(epoch):
if (log_img):
print('Saved Epoch', i)
generator.save_weights(dataset + '_gen.h5')
discriminator.save_weights(dataset + '_dis.h5')
plot_output(dataset, i)
for j in tqdm(range(batch_count)):
# Input for the generator
noise_input = np.random.rand(batch_size, noise_size)
if dataset == 'cifar10' or dataset == 'mnist' or dataset == 'o-faces' or dataset == 'cifar100':
image_batch = data[np.random.randint(0,
data.shape[0],
size=batch_size)]
else:
batch_files = data[j * batch_size:(j + 1) * batch_size]
batch = [
scipy.misc.imread(batch_file).astype(np.float)
for batch_file in batch_files
]
image_batch = np.array(batch).astype(np.float32)
image_batch = (image_batch - 127.5) / 127.5
# these are the predicted images from the generator
predictions = generator.predict(noise_input,
batch_size=batch_size)
#print(predictions.shape)
#print(image_batch.shape)
# the discriminator takes in the real images and the generated images
X = np.concatenate([predictions, image_batch])
# labels for the discriminator
y_discriminator = [0] * batch_size + [1] * batch_size
# Let's train the discriminator
discriminator.trainable = True
discriminator.train_on_batch(X, y_discriminator)
# Let's train the generator
noise_input = np.random.rand(batch_size, noise_size)
y_generator = [1] * batch_size
discriminator.trainable = False
gan.train_on_batch(noise_input, y_generator)
def train(dataset, alpha, beta, thr):
num_cores = multiprocessing.cpu_count()
alpha = alpha
beta = beta
thr = thr
w_lim = 1
c = 5
batch_budget = 250
bnn = 0
al_method = 'bvssb'
dynamic = 0
budget = 0
epochs_init = 4
epochs = 3
batch_size = 64
noise_ratio = 30
NUM_CLASSES = {
'mnist': 10,
'svhn': 10,
'cifar-10': 10,
'cifar-100': 100,
'celeb': 20
}
dataset = dataset
init_noise_ratio = 0
data_ratio = 1.666666
X_train, y_train, X_test, y_test, x_val, y_val, un_selected_index = get_data(
dataset, init_noise_ratio, data_ratio, random_shuffle=False)
image_shape = X_train.shape[1:]
model = get_model(bnn,
dataset,
input_tensor=None,
input_shape=image_shape,
num_classes=NUM_CLASSES[dataset])
optimizer = SGD(lr=0.1, decay=0.002, momentum=0.9)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False, # randomly flip images
)
datagen.fit(X_train)
h_quality = model.fit(datagen.flow(X_train, y_train,
batch_size=batch_size),
steps_per_epoch=X_train.shape[0] // batch_size,
epochs=epochs_init,
validation_data=(X_test, y_test),
workers=num_cores)
model_quality = clone_model(model)
model_quality.set_weights(model.get_weights())
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(-1, 28, 28, 1)
X_test = X_test.reshape(-1, 28, 28, 1)
X_train = X_train / 255.0
X_test = X_test / 255.0
means = X_train.mean(axis=0)
# std = np.std(X_train)
X_train = (X_train - means) # / std
X_test = (X_test - means) # / std
# they are 2D originally in cifar
y_train = y_train.ravel()
y_test = y_test.ravel()
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
y_train = np_utils.to_categorical(y_train, NUM_CLASSES[dataset])
y_test = np_utils.to_categorical(y_test, NUM_CLASSES[dataset])
epochs_training = epochs
training_noise_level = noise_ratio
loss = 0
used_budget = 0
total_clean_num = 0
n_classes = NUM_CLASSES[dataset]
# training_steps indicate how many data used in each batch
training_steps = 1000
statistics_list = [[] for _ in range(21)]
steps = int(np.ceil(len(un_selected_index) / float(training_steps)))
for i in np.arange(steps):
print("chunk:", i)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
if i == 0:
sub_un_selected_list = un_selected_index[0:training_steps]
elif i != steps - 1:
sub_un_selected_list = un_selected_index[i *
training_steps:(i + 1) *
training_steps]
else:
sub_un_selected_list = un_selected_index[i * training_steps:]
if i != 0:
model_quality = clone_model(model)
model_quality.set_weights(model.get_weights())
X_in_iteration = X_train[sub_un_selected_list]
y_true_iteration = y_train[sub_un_selected_list]
y_noisy_iteration, noisy_idx = inject_noise(
dataset, y_train[sub_un_selected_list], training_noise_level)
y_predict, predict_prob, y_predict_label_second = BNN_label_predict(
X_in_iteration, model, n_classes)
clean_list, clean_pred_list, noisy_list, FN, TN, TP, FP, maxProbTP, maxProbFP = select_clean_noisy(
X_in_iteration, y_noisy_iteration, y_true_iteration, y_predict,
y_predict_label_second, predict_prob, model)
oracle_list = []
if batch_budget > 0:
num_al = len(noisy_list)
print("batch_budget:", batch_budget)
batch_list = range(0, training_steps)
n = NUM_CLASSES[dataset]
spent_s = 0
spent_w = 0
inf_ind, inf = BNN_active_selection(predict_prob, noisy_list,
al_method, num_al)
loss = model_reliability(model, x_val, y_val)
strong_list, weak_list, spent_s, spent_w, first_yes, weak_questions, mean_w_q, unique_pic, true_label_rank = label_decider(
noisy_list, y_true_iteration, y_predict, predict_prob, n, inf_ind,
inf, loss, batch_budget, spent_s, spent_w, w_lim, c, alpha, beta,
thr)
statistics_list[0].append(len(strong_list))
statistics_list[1].append(len(weak_list))
statistics_list[2].append(first_yes)
statistics_list[3].append(spent_w)
statistics_list[4].append(mean_w_q)
statistics_list[5].append(unique_pic)
statistics_list[6].append(num_al)
print("spent_s:", spent_s, "spent_s/c:", spent_s / c)
print("spent_w:", spent_w, "weak yes:", len(weak_list))
oracle_list = np.append(strong_list, weak_list)
if len(weak_list) == 0:
oracle_list = strong_list
if len(strong_list) == 0:
oracle_list = weak_list
y_noisy_iteration[oracle_list] = y_true_iteration[oracle_list]
training_list = np.append(clean_list, oracle_list)
x_train_iteration = X_in_iteration[training_list]
y_train_iteration = y_noisy_iteration[training_list]
print("train data shape:", x_train_iteration.shape)
h_training_epoch_quality = model.fit_generator(
datagen.flow(x_train_iteration,
y_train_iteration,
batch_size=batch_size),
steps_per_epoch=y_train_iteration.shape[0] // batch_size + 1,
epochs=epochs_training,
validation_data=(X_test, y_test))
h_quality = combine_result(h_quality, h_training_epoch_quality)
if i != 0 and h_quality.history['val_accuracy'][
-epochs_training - 1] - np.min(h_quality.history[
'val_accuracy'][-epochs_training:]) > 0.2:
model = clone_model(model_quality)
model.set_weights(model_quality.get_weights())
s_bnn = 0
val_accc = h_quality.history['val_accuracy']
accc = h_quality.history['accuracy']
val_losss = h_quality.history['val_loss']
losss = h_quality.history['loss']
for i in range(0, 10):
s_bnn = s_bnn + val_accc[-epochs_training * i - 1]
print(val_accc[-epochs_training * i - 1])
average_valacc = s_bnn / 10
print("Final Acc:", average_valacc)
statistics_list[14].append(val_accc)
statistics_list[15].append(accc)
statistics_list[16].append(val_losss)
statistics_list[17].append(losss)
end = time.time()
statistics_list[18].append(end - start)
print("timing: ", end - start)
#################################### save ########################################
with open(
'WSstat_alpha' + str(alpha) + '+beta' + str(beta) + '+threshold' +
str(thr) + '.pickle', 'wb') as file_pi:
pickle.dump(statistics_list, file_pi)
return average_valacc
# In[6]:
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.linear_model import LogisticRegression, LogisticRegressionCV
# ## Pre-Processed Data
# In[19]:
from datasets import get_data
X, X_0, Y = get_data()
# ### Principle Components Analysis
# In[21]:
n_components = [3, 4, 5]
pca = PCA()
logistic = LogisticRegression()
pipe = Pipeline(steps=[('pca', pca), ('logistic', logistic)])
estimator = GridSearchCV(pipe,
dict(
pca__n_components=n_components,
