def main():
parser = argparse.ArgumentParser()
parser.add_argument("-batch_size", type=int, default=30)
parser.add_argument("-N", type=int, default=3)
parser.add_argument("-embedding_dim", type=int, default=512)
parser.add_argument("-lr", type=float, default=1e-3)
parser.add_argument("-weight_decay", type=float, default=1e-5)
parser.add_argument("-iterator", type=int, default=30)
args = parser.parse_args()
train_batch, val, test, vocab = dataloader(batch_size=args.batch_size,
memory_size=5,
task=1,
joint=False,
tenK=False)
model = MemoryNetwork(vocab_size=len(vocab),
d=args.embedding_dim,
N=args.N)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
train(model=model,
iterator=args.iterator,
optimizer=optimizer,
criterion=criterion,
train_loader=train_batch,
test_loader=None)
def fit(self):
# Load data
self.dl = dataloader(dataset=self.dataset,
norm_method=self.norm_method,
val_fold=self.val_fold,
crop=self.crop,
inv_thresh=self.inv_thresh,
custom_data=self.custom_data,
verbose=2)
# Init network
self.dtc = DTC(n_clusters=self.n_clusters,
input_dim=1,
timesteps=self.dl.data_train.shape[1],
n_filters=self.n_filters,
kernel_size=self.kernel_size,
strides=self.strides,
pool_size=self.pool_size,
n_units=self.n_units,
alpha=self.alpha,
dist_metric=self.dist_metric,
cluster_init=self.cluster_init,
heatmap=self.heatmap)
# Initialize and compile model
self.dtc.initialize()
self.dtc.model.summary()
self.dtc.compile(gamma=self.gamma,
optimizer=self.optimizer,
initial_heatmap_loss_weight=self.initial_heatmap_loss_weight,
final_heatmap_loss_weight=self.final_heatmap_loss_weight)
# Pretrain
if (self.pretrain_epochs > 0):
self.dtc.pretrain(X=self.dl.data_train,
optimizer=self.optimizer,
epochs=self.pretrain_epochs,
batch_size=self.batch_size,
save_dir=self.save_dir)
# Initialize clusters
self.dtc.init_cluster_weights(self.dl.data_train)
# Fit model
self.dtc.fit(X_train=self.dl.data_train,
y_train=self.dl.label_train,
X_val=self.dl.data_val,
y_val=self.dl.label_val,
epochs=self.epochs,
eval_epochs=self.eval_epochs,
save_epochs=self.save_epochs,
batch_size=self.batch_size,
tol=0.001,
patience=5,
finetune_heatmap_at_epoch=self.finetune_heatmap_at_epoch,
save_dir=self.save_dir)
self.model = self.dtc.model
def train(self, task_id, trainData, trainLabels, trainLabelsVectors):
self.discriminator = self.get_discriminator(task_id)
self.model_optimizer = self.get_model_optimizer(task_id)
self.d_optimizer = self.get_dis_optimizer()
train_data, train_label, train_attr, no_steps = utils.dataloader(trainData, trainLabels, trainLabelsVectors, self.args.train_batch_size)
for e in range(self.args.num_epochs):
loss = self.train_epoch(task_id, train_data, train_label, train_attr, no_steps)
print('epoch', e + 1, 'Total loss:', loss)
def __init__(self, args):
self.initial_lr = args.learning_rate
self.lr = args.learning_rate
self.test_only = args.test_only
self.dump_statistics = args.dump_statistics
self.modelName = args.model
self.experiment = args.experiment
self.log_path = args.log_path
self.save_path = args.save_path
if not os.path.isdir(self.log_path):
os.makedirs(self.log_path)
self.logger = Logger(
'%s/%s_%s.csv' % (self.log_path, self.modelName, args.experiment),
'epoch, time, learning_rate, tr_loss, tr_acc, val_loss, val_acc')
self.progress_bar = ProgressBar()
self.chrono = Chrono()
self.trainset, self.testset, self.trainloader, self.testloader = dataloader(
)
print('==> Building model..')
self.ae = AutoEncoder()
self.model = getattr(models, self.modelName)()
if self.modelName == 'bit':
self.model.load_from(
numpy.load('./state_dicts/%s.npz' % self.modelName))
if torch.cuda.is_available():
self.ae = torch.nn.DataParallel(self.ae)
self.model = torch.nn.DataParallel(self.model)
torch.backends.cudnn.benchmark = True
self.optimizer = torch.optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=0.9)
self.load_ae()
if args.resume or self.test_only or self.dump_statistics:
self.load()
self.criterion = torch.nn.CrossEntropyLoss()
self.criterion = get_torch_vars(self.criterion, False)
self.ae = get_torch_vars(self.ae, False)
self.model = get_torch_vars(self.model, False)
def train(self):
opt = tf.keras.optimizers.Adam(amsgrad=True)
model = self.model
#apply binary cross entropy for the loss of confidence scores and custom smooth l1 loss for regressing the anchor coordinates.
model.compile(loss=['binary_crossentropy', smooth_l1_loss],
optimizer=opt)
X, Y = dataloader()
for epoch in range(self.nb_epoch):
res = model.fit(x=X,
y=Y,
batch_size=self.batch_size,
initial_epoch=epoch,
epochs=epoch + 1)
if epoch % 100 == 0:
model.save_weights(
os.path.join(config.checkpoint_path, str(epoch)))
def __init__(self, args):
# Parametres
self.epoch = args.epoch
self.sample_num = 100
self.batch_size = args.batch_size
self.input_size = args.input_size
self.z_dim = 62
self.result_dir = 'results'
self.save_dir = 'models'
self.model_name = 'WGAN'
self.dataset = args.dataset
self.c = 0.01 # clipping value
self.n_critic = 5 # the number of iterations of the critic per generator iteration
# load dataset
self.data_loader = utils.dataloader(self.input_size, self.batch_size,
self.dataset)
data = self.data_loader.__iter__().__next__()[0]
# networks init
self.G = generator(input_dim=self.z_dim,
output_dim=data.shape[1],
input_size=self.input_size)
self.D = discriminator(input_dim=data.shape[1],
output_dim=1,
input_size=self.input_size)
self.G_optimizer = optim.Adam(self.G.parameters(),
lr=args.lrG,
betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(),
lr=args.lrD,
betas=(args.beta1, args.beta2))
self.G.cpu()
self.D.cpu()
print('---------- Networks architecture -------------')
utils.print_network(self.G)
utils.print_network(self.D)
print('-----------------------------------------------')
# fixed noise
self.sample_z_ = torch.rand((self.batch_size, self.z_dim))
self.sample_z_ = self.sample_z_.cpu()
def fit(self):
# Prepare data
self.dl = dataloader(dataset=self.dataset,
norm_method=self.norm_method,
val_fold=self.val_fold,
crop=self.crop,
inv_thresh=self.inv_thresh,
custom_data=self.custom_data,
format='Squashed',
verbose=2)
# Init network
self.input_dim = 1
self.timesteps = int(self.dl.data_train.shape[1])
self.build_ae()
# Prepare callbacks
early_stop = EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
patience=5,
min_delta=0.000)
filepath = "model-ae-epoch-{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
mode='min',
verbose=1,
save_best_only=False,
period=10)
# Fit model
self.history = self.autoencoder.fit(
x=self.dl.data_train,
y=self.dl.data_train,
validation_data=[self.dl.data_val, self.dl.data_val],
nb_epoch=self.epochs,
batch_size=self.batch_size,
verbose=1,
callbacks=[early_stop, checkpoint]).history
def fit(self):
# Prepare data
self.dl = dataloader(dataset=self.dataset,
norm_method=self.norm_method,
val_fold=self.val_fold,
crop=self.crop,
inv_thresh=self.inv_thresh,
custom_data=self.custom_data,
verbose=2)
# Init network
self.autoencoder, self.encoder, self.decoder = temporal_autoencoder_v3(input_dim=1,
timesteps=self.dl.data_train.shape[1],
n_filters=self.n_filters,
kernel_size=self.kernel_size,
strides=self.strides,
pool_size=self.pool_size,
n_units=self.n_units,
latent_dim=self.latent_dim)
# DTC model (autoencoder only)
self.model = Model(inputs=self.autoencoder.input, outputs=self.autoencoder.output)
# Compile model
self.model.compile(loss=self.loss, optimizer=self.optimizer)
# Prepare callbacks
early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5, min_delta=0.000)
filepath = "model-ae-epoch-{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', mode='min', verbose=1, save_best_only=False, period=10)
# Fit model
self.history = self.model.fit(x=self.dl.data_train,
y=self.dl.data_train,
validation_data=[self.dl.data_val, self.dl.data_val],
nb_epoch=self.epochs,
batch_size=self.batch_size,
verbose=2,
callbacks=[early_stop, checkpoint]).history
def __init__(self, args):
# Parametres
self.epoch = args.epoch
self.sample_num = 100
self.batch_size = args.batch_size
self.input_size = args.input_size
self.z_dim = 62
self.result_dir = 'results'
self.save_dir = 'models'
self.dataset = args.dataset
self.model_name = 'GAN'
# Chargement des données
self.data_loader = utils.dataloader(self.input_size, self.batch_size, self.dataset)
data = self.data_loader.__iter__().__next__()[0]
# Initialisation du générateur et du discriminant
self.G = generator(input_dim=self.z_dim, output_dim=data.shape[1], input_size=self.input_size)
self.D = discriminator(input_dim=data.shape[1], output_dim=1, input_size=self.input_size)
# Optimisation avec Adam
self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.lrG, betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.lrD, betas=(args.beta1, args.beta2))
self.G.cpu()
self.D.cpu()
self.BCE_loss = nn.BCELoss().cpu()
print('---------- Networks architecture -------------')
utils.print_network(self.G)
utils.print_network(self.D)
print('-----------------------------------------------')
# Bruit
self.sample_z_ = torch.rand((self.batch_size, self.z_dim))
self.sample_z_ = self.sample_z_.cpu()
def main(args):
model = AutoEncoder()
use_gpu = torch.cuda.is_available()
if use_gpu:
print('cuda is available!')
model.cuda()
weight_file = args.path_weight_file
model = AutoEncoder()
model.load_state_dict(
torch.load(weight_file, map_location=lambda storage, loc: storage))
test_dataset = dataloader('dogs_cats', 'test')
test_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=args.batch_size, shuffle=False)
images, _ = iter(test_loader).next()
images = Variable(images, volatile=True)
imshow(torchvision.utils.make_grid(images.data[:25], nrow=5))
outputs = model(images)
imshow(torchvision.utils.make_grid(outputs.data[:25], nrow=5))
def __init__(self, args):
self.initial_lr = args.learning_rate
self.lr = args.learning_rate
self.test_only = args.test_only
self.dump_statistics = args.dump_statistics
self.modelName = "malconv"
self.experiment = args.experiment
self.log_path = args.log_path
self.save_path = args.save_path
if not os.path.isdir(self.log_path):
os.makedirs(self.log_path)
self.logger = Logger('%s/%s_%s.csv' % (self.log_path, self.modelName, args.experiment),
'epoch, time, learning_rate, tr_loss, tr_acc, val_loss, val_acc')
self.progress_bar = ProgressBar()
self.chrono = Chrono()
self.trainset, self.testset, self.trainloader, self.testloader = dataloader(args.first_n_byte)
print('==> Building model..')
self.model = MalConv(input_length=args.first_n_byte)
if torch.cuda.is_available():
self.model = torch.nn.DataParallel(self.model)
torch.backends.cudnn.benchmark = True
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)
if args.resume or self.test_only or self.dump_statistics:
self.load()
self.criterion = torch.nn.BCELoss()
self.criterion = get_torch_vars(self.criterion, False)
self.model = get_torch_vars(self.model, False)
def process_and_train_FL(model: BasicModel, opt: config.Option):
# preparing saving files.
save_path = os.path.join(opt.save_dir + '/model_file',
opt.save_model_name).replace('\\', '/')
print("model file save path: ", save_path)
if not os.path.exists(save_path):
os.makedirs(save_path)
msger = messager(
save_path=save_path,
types=[
'train_data_file', 'val_data_file', 'test_data_file',
'load_model_name', 'save_model_name', 'trainset_loss_type',
'testset_loss_type', 'class_num_ratio'
],
json_name='train_information_msg_' +
time.strftime('%m{}%d{}_%H:%M'.format('月', '日')) + '.json')
msger.record_message([
opt.train_data_file, opt.val_data_file, opt.test_data_file,
opt.load_model_name, opt.save_model_name, opt.train_loss_type,
opt.testset_loss_type, opt.class_num_ratio
])
msger.save_json()
# train data loading
print('-----Data Loading-----')
if opt.BERT:
dataloader_train = dataloader_BERT(opt.train_data_file,
opt.wordvec_file,
opt.rel2id_file,
opt.similarity_file,
opt.same_level_pair_file,
max_len=opt.max_len,
random_init=opt.random_init,
seed=opt.seed)
dataloader_val = dataloader_BERT(opt.val_data_file,
opt.wordvec_file,
opt.rel2id_file,
opt.similarity_file,
max_len=opt.max_len)
dataloader_test = dataloader_BERT(opt.test_data_file,
opt.wordvec_file,
opt.rel2id_file,
opt.similarity_file,
max_len=opt.max_len)
else:
dataloader_train = dataloader(opt.train_data_file,
opt.wordvec_file,
opt.rel2id_file,
opt.similarity_file,
opt.same_level_pair_file,
max_len=opt.max_len,
random_init=opt.random_init,
seed=opt.seed,
data_type=opt.data_type)
dataloader_val = dataloader(opt.val_data_file,
opt.wordvec_file,
opt.rel2id_file,
opt.similarity_file,
max_len=opt.max_len,
data_type=opt.data_type)
dataloader_test = dataloader(opt.test_data_file,
opt.wordvec_file,
opt.rel2id_file,
opt.similarity_file,
max_len=opt.max_len,
data_type=opt.data_type)
word_emb_dim = dataloader_train._word_emb_dim_()
word_vec_mat = dataloader_train._word_vec_mat_() # numpy.array float32
print('word_emb_dim is {}'.format(word_emb_dim))
# compile model
print('-----Model Initializing-----')
if opt.BERT != True:
model.set_embedding_weight(word_vec_mat)
if opt.load_model_name is not None:
model.load_model(opt.load_model_name)
os.environ["CUDA_VISIBLE_DEVICES"] = opt.gpu
if torch.cuda.is_available():
torch.cuda.set_device(int(opt.gpu))
model, cuda_flag = tl.cudafy(model)
if not cuda_flag:
print("There is no gpu,use default cpu")
count = tl.count_parameters(model)
print("num of parameters:", count)
# if the datasets are imbalanced such as nyt_su or trex , we load all test/dev data to perform open setting
print('-----Validation Data Preparing-----')
try:
opt.data_type.index('imbalance')
print("try load all imbalance dev data!")
val_data, val_data_label = dataloader_val._data_()
except:
print("load part of data")
if opt.data_type.startswith('fewrel'):
val_data, val_data_label = dataloader_val._part_data_(
100
) # 16 relation classes in validation data,each class has 100 sample in fewrel
else:
# other data sets has the problem of label imbalance
val_data, val_data_label = dataloader_val._part_data_(
100
) # for nyt_fb :sample 5 instance per relation, will get 490 dev instance
print("-------Test Data Preparing--------")
try:
opt.data_type.index('imbalance')
print("try load all imbalance test data!")
test_data, test_data_label = dataloader_test._data_()
except:
print("load part of data")
if opt.data_type.startswith('fewrel'):
test_data, test_data_label = dataloader_test._data_()
else:
test_data, test_data_label = dataloader_test._data_(
100) # sample as the dev setting
print("val_data:", len(val_data))
print("val_data_label:", len(set(val_data_label)))
print("test_data:", len(test_data))
print("test_data_label:", len(set(test_data_label)))
# intializing parameters
batch_num_list = opt.batch_num
msger_cluster = messager(
save_path=save_path,
types=[
'method', 'temp_epoch', 'temp_batch_num', 'temp_batch_size',
'temp_lr', 'NMI', 'F1', 'precision', 'recall', 'msg'
],
json_name='Validation_cluster_msg_' +
time.strftime('%m{}%d{}_%H:%M'.format('月', '日')) + '.json')
if opt.record_test:
msger_test = messager(
save_path=save_path,
types=[
'temp_globle_step', 'temp_batch_size', 'temp_learning_rate',
'NMI', 'F1', 'precision', 'recall', 'msg'
],
json_name='Test_cluster_msg_' +
time.strftime('%m{}%d{}_%H:%M'.format('月', '日')) + '.json')
if opt.whether_visualize:
loger = SummaryWriter(comment=opt.save_model_name)
else:
loger = None
best_batch_step = 0
best_epoch = 0
batch_size_chose = -1
print_flag = opt.print_losses
best_validation_f1 = 0
best_test_f1 = 0
loss_list = []
global_step = 0
for epoch in range(opt.epoch_num):
print('------epoch {}------'.format(epoch))
print('max batch num to train is {}'.format(batch_num_list[epoch]))
loss_reduce = 10000.
early_stop_record = 0
for i in range(1, batch_num_list[epoch] + 1):
global_step += 1
loss_list = model.train_self(opt,
dataloader_train,
loss_list,
loger,
batch_chose=batch_size_chose,
global_step=global_step,
temp_epoch=epoch)
# print loss & record loss
if i % 100 == 0:
ave_loss = sum(loss_list) / 100.
print('temp_batch_num: ', i, ' total_batch_num: ',
batch_num_list[epoch], " ave_loss: ", ave_loss,
' temp learning rate: ', opt.lr[opt.lr_chose])
# empty the loss list
loss_list = []
# visualize
if opt.whether_visualize:
loger.add_scalar('all_epoch_loss',
ave_loss,
global_step=global_step)
# early stop
if opt.early_stop is not None:
if ave_loss < loss_reduce:
early_stop_record = 0
loss_reduce = ave_loss
else:
early_stop_record += 1
if early_stop_record == opt.early_stop:
print(
"~~~~~~~~~ The loss can't be reduced in {} step, early stop! ~~~~~~~~~~~~"
.format(opt.early_stop * 100))
cluster_result, cluster_msg, cluster_center, features = K_means_BERT(
test_data, model.pred_vector, test_data_label,
opt) if opt.BERT else K_means(
test_data, model.pred_vector,
len(np.unique(test_data_label)), opt)
cluster_test_b3 = ClusterEvaluation(
test_data_label,
cluster_result).printEvaluation(extra_info=True,
print_flag=True)
print("learning rate decay num:", opt.lr_decay_num)
print("learning rate decay step:", opt.lr_decay_record)
print("best_epoch:", best_epoch)
print("best_step:", best_batch_step)
print("best_batch_size:", best_batch_size)
print("best_cluster_eval_b3:", best_validation_f1)
print("seed:", opt.seed)
# clustering & validation
if i % 200 == 0:
print(opt.save_model_name, 'epoch:', epoch)
with torch.no_grad():
# fewrel -> K-means ; nyt+su -> Mean-Shift
if opt.dataset.startswith("fewrel"):
print("chose k-means >>>")
F_score = -1.0
best_cluster_result = None
best_cluster_msg = None
best_cluster_center = None
best_features = None
best_cluster_eval_b3 = None
for iterion in range(opt.eval_num):
K_num = opt.K_num if opt.K_num != 0 else len(
np.unique(val_data_label))
cluster_result, cluster_msg, cluster_center, features = K_means_BERT(
val_data, model.pred_vector, val_data_label,
opt) if opt.BERT else K_means(
val_data, model.pred_vector, K_num, opt)
cluster_eval_b3 = ClusterEvaluation(
val_data_label,
cluster_result).printEvaluation(
print_flag=False)
if F_score < cluster_eval_b3['F1']:
F_score = cluster_eval_b3['F1']
best_cluster_result = cluster_result
best_cluster_msg = cluster_msg
best_cluster_center = cluster_center
best_features = features
best_cluster_eval_b3 = cluster_eval_b3
cluster_result = best_cluster_result
cluster_msg = best_cluster_msg
cluster_center = best_cluster_center
features = best_features
cluster_eval_b3 = best_cluster_eval_b3
else:
print("chose mean-shift >>>")
cluster_result, cluster_msg, cluster_center, features = mean_shift_BERT(
val_data, model.pred_vector, val_data_label,
opt) if opt.BERT else mean_shift(
val_data, model.pred_vector, opt)
cluster_eval_b3 = ClusterEvaluation(
val_data_label,
cluster_result).printEvaluation(print_flag=False,
extra_info=True)
NMI_score = normalized_mutual_info_score(
val_data_label, cluster_result)
print("NMI:{} ,F1:{} ,precision:{} ,recall:{}".format(
NMI_score,
cluster_eval_b3['F1'],
cluster_eval_b3['precision'],
cluster_eval_b3['recall'],
))
msger_cluster.record_message([
opt.select_cluster, epoch, i,
opt.batch_size[batch_size_chose], model.lr, NMI_score,
cluster_eval_b3['F1'], cluster_eval_b3['precision'],
cluster_eval_b3['recall'], cluster_msg
])
msger_cluster.save_json()
two_f1 = cluster_eval_b3['F1']
if two_f1 > best_validation_f1: # acc
if opt.record_test == False:
model.save_model(model_name=opt.save_model_name,
global_step=global_step)
best_batch_step = i
best_epoch = epoch
best_batch_size = opt.batch_size[batch_size_chose]
best_validation_f1 = two_f1
if opt.whether_visualize:
loger.add_embedding(features,
metadata=val_data_label,
label_img=None,
global_step=global_step,
tag='ground_truth',
metadata_header=None)
loger.add_embedding(features,
metadata=cluster_result,
label_img=None,
global_step=global_step,
tag='prediction',
metadata_header=None)
loger.add_scalar('all_epoch_NMI',
NMI_score,
global_step=global_step)
loger.add_scalar('all_epoch_F1',
cluster_eval_b3['F1'],
global_step=global_step)
loger.add_scalar('all_epoch_precision',
cluster_eval_b3['precision'],
global_step=global_step)
loger.add_scalar('all_epoch_recall',
cluster_eval_b3['recall'],
global_step=global_step)
if opt.record_test:
if opt.dataset.startswith("fewrel"):
cluster_result, cluster_msg, cluster_center, features = K_means_BERT(
test_data, model.pred_vector, test_data_label,
opt) if opt.BERT else K_means(
test_data, model.pred_vector,
len(np.unique(test_data_label)), opt)
cluster_test_b3 = ClusterEvaluation(
test_data_label,
cluster_result).printEvaluation(
print_flag=False)
else:
cluster_result, cluster_msg, cluster_center, features = mean_shift_BERT(
test_data, model.pred_vector, test_data_label,
opt) if opt.BERT else mean_shift(
test_data, model.pred_vector, opt)
cluster_test_b3 = ClusterEvaluation(
test_data_label,
cluster_result).printEvaluation(
print_flag=False, extra_info=True)
msger_test.record_message([
global_step, opt.batch_size[batch_size_chose],
opt.lr[opt.lr_chose], NMI_score,
cluster_test_b3['F1'],
cluster_test_b3['precision'],
cluster_test_b3['recall'], cluster_msg
])
msger_test.save_json()
print('test messages saved.')
if cluster_test_b3['F1'] > best_test_f1:
model.save_model(model_name=opt.save_model_name,
global_step=global_step)
best_batch_step = i
best_epoch = epoch
best_batch_size = opt.batch_size[batch_size_chose]
best_test_f1 = cluster_test_b3['F1']
model.lr_decay(opt)
opt.lr_decay_record.append(global_step)
print('End: The model is:', opt.save_model_name, opt.train_loss_type,
opt.testset_loss_type)
if opt.dataset.startswith("fewrel"):
print('\n-----K-means Clustering test-----')
best_test_b3, NMI_score = k_means_cluster_evaluation(
model, opt, test_data, test_data_label, loger)
else:
print("\n-----------Mean_shift Clustering test:---------------")
model.load_model(opt.save_model_name + "_best.pt")
cluster_result_ms, cluster_msg_ms, _, _ = mean_shift_BERT(
test_data, model.pred_vector, test_data_label,
opt) if opt.BERT else mean_shift(test_data, model.pred_vector, opt)
cluster_eval_b3_ms = ClusterEvaluation(
test_data_label,
cluster_result_ms).printEvaluation(print_flag=opt.print_losses,
extra_info=True)
NMI_score_ms = normalized_mutual_info_score(test_data_label,
cluster_result_ms)
best_test_b3 = cluster_eval_b3_ms
NMI_score = NMI_score_ms
if opt.whether_visualize:
loger.add_scalar('test_NMI_MeanShift', NMI_score_ms, global_step=0)
loger.add_scalar('test_F1_MeanShift',
cluster_eval_b3_ms['F1'],
global_step=0)
print("learning rate decay num:", opt.lr_decay_num)
print("learning rate decay step:", opt.lr_decay_record)
print("best_epoch:", best_epoch)
print("best_step:", best_batch_step)
print("best_batch_size:", best_batch_size)
print("best_cluster_eval_b3:", best_validation_f1)
print("best_cluster_test_b3:", best_test_b3)
print("best_NMI_score:", NMI_score)
print("seed:", opt.seed)
def train(classifier, segmenter, epochs=10):
data_loader = dataloader()
loss = torch.nn.CrossEntropyLoss()
gen_opt = torch.optim.Adam(segmenter.parameters())
clas_opt = torch.optim.Adam(classifier.parameters())
for i in range(epochs):
for j, (imgs, ground_truths) in enumerate(data_loader):
#Extract class label from ground truth label
lbls = []
for k in range(len(ground_truths)):
lbl = ground_truths[k]
classes, mask = torch.unique(lbl, return_inverse=True)
classes *= 255
max = 0
#Choose the class that has the largest area
for l in range(len(classes)):
if classes[l] != 0 and classes[l] != 255 and torch.sum(
mask == l) > max:
max = l
lbls.append(classes[max])
imgs = imgs.cuda()
lbls = torch.from_numpy(np.array(lbls)).long().cuda() - 1
#Obtain the Guided Backprop and GradCAM
gc = grad_cam(classifier, imgs)
gbp = guided_bp(classifier, imgs)
#Input to the segmenter is the original image + gc + gbp (5 channels)
seg_input = torch.cat((imgs, gc, gbp), 1).cuda()
#Invert the output mask to leave only the background
seg_output = segmenter(seg_input)[:, lbls[0], :, :]
masks = (seg_output < 0.9).type(torch.cuda.FloatTensor)
seg_imgs = []
for k in range(imgs.shape[0]):
seg_imgs.append(imgs[k] * masks[k, k])
#Segmentor tries to increase the classifier loss after extracting the target class
seg_imgs = torch.stack(seg_imgs)
s_loss = -loss(classifier(seg_imgs), lbls)
gen_opt.zero_grad()
s_loss.backward()
gen_opt.step()
#Classifier tries to decrease loss on both original images and segmented images
c_loss = loss(classifier(imgs), lbls) + loss(
classifier(seg_imgs), lbls)
clas_opt.zero_grad()
c_loss.backward()
clas_opt.step()
if j % 20 == 0:
print(
"Epoch {} Iter {} - Segmentation Loss: {} Classificaton Loss: {}"
.format(i, j, s_loss * -1, c_loss))
return classifier, segmenter
def test_stardardize(self):
x, y = dataloader(mode='train', reduced=False)
x = standardize(x)
npt.assert_array_almost_equal(np.mean(x, axis=0), np.zeros(30))
npt.assert_array_almost_equal(np.std(x, axis=0), np.ones(30))
def train(train_x, train_y, learning_rate=0.1, num_epochs=50, batch_size=1):
# Flatten input (num_samples, 28, 28) -> (num_samples, 784)
x = train_x.reshape(train_x.shape[0], -1)
num_samples = x.shape[0]
# Turn labels into their one-hot representations
y = one_hot_encoder(train_y)
# Make a data loader
trainloader = dataloader(x, y, batch_size=batch_size, shuffle=True)
# Initialize weights
w1, b1 = initialize_weight((784, 256), bias=True)
w2, b2 = initialize_weight((256, 10), bias=True)
loss_history = []
for epoch in range(1, num_epochs + 1):
print("Epoch {}/{}\n===============".format(epoch, num_epochs))
batch_loss = 0
acc = 0
for inputs, labels in trainloader:
# Number of samples per batch
m = inputs.shape[0]
# Forward Prop
h1 = np.dot(inputs, w1) + b1
a1 = sigmoid(h1)
h2 = np.dot(a1, w2) + b2
a2 = softmax(h2)
out = a2
# Cross Entropy Loss
batch_loss += cross_entropy_loss(
out,
labels.argmax(axis=1).reshape(m, 1))
# Compute Accuracy
pred = np.argmax(out, axis=1)
pred = pred.reshape(pred.shape[0], 1)
acc += np.sum(pred == labels.argmax(axis=1).reshape(m, 1))
# Backward Prop
dh2 = a2 - labels
dw2 = (1 / m) * np.dot(a1.T, dh2)
db2 = (1 / m) * np.sum(dh2, axis=0, keepdims=True)
dh1 = np.dot(dh2, w2.T) * sigmoid_prime(a1)
dw1 = (1 / m) * np.dot(inputs.T, dh1)
db1 = (1 / m) * np.sum(dh1, axis=0, keepdims=True)
# Weight (and bias) update
w1 -= learning_rate * dw1
b1 -= learning_rate * db1
w2 -= learning_rate * dw2
b2 -= learning_rate * db2
loss_history.append(batch_loss / num_samples)
print("Loss: {:.6f}".format(batch_loss / num_samples))
print("Accuracy: {:.2f}%\n".format(acc / num_samples * 100))
return w1, b1, w2, b2, loss_history
def __init__(self, args, SUPERVISED=True):
# Parametres
self.epoch = args.epoch
self.batch_size = args.batch_size
self.save_dir = 'models'
self.result_dir = 'results'
self.dataset = args.dataset
self.model_name = 'InfoGAN'
self.input_size = args.input_size
self.z_dim = 62
self.SUPERVISED = SUPERVISED # if it is true, label info is directly used for code
self.len_discrete_code = 10 # categorical distribution (i.e. label)
self.len_continuous_code = 2 # gaussian distribution (e.g. rotation, thickness)
self.sample_num = self.len_discrete_code ** 2
# load dataset
self.data_loader = utils.dataloader(self.input_size, self.batch_size, self.dataset)
data = self.data_loader.__iter__().__next__()[0]
# networks init
self.G = generator(input_dim=self.z_dim, output_dim=data.shape[1], input_size=self.input_size, len_discrete_code=self.len_discrete_code, len_continuous_code=self.len_continuous_code)
self.D = discriminator(input_dim=data.shape[1], output_dim=1, input_size=self.input_size, len_discrete_code=self.len_discrete_code, len_continuous_code=self.len_continuous_code)
self.G_optimizer = optim.Adam(self.G.parameters(), lr=args.lrG, betas=(args.beta1, args.beta2))
self.D_optimizer = optim.Adam(self.D.parameters(), lr=args.lrD, betas=(args.beta1, args.beta2))
self.info_optimizer = optim.Adam(itertools.chain(self.G.parameters(), self.D.parameters()), lr=args.lrD, betas=(args.beta1, args.beta2))
self.G.cpu()
self.D.cpu()
self.BCE_loss = nn.BCELoss().cpu()
self.CE_loss = nn.CrossEntropyLoss().cpu()
self.MSE_loss = nn.MSELoss().cpu()
print('---------- Networks architecture -------------')
utils.print_network(self.G)
utils.print_network(self.D)
print('-----------------------------------------------')
# fixed noise & condition
self.sample_z_ = torch.zeros((self.sample_num, self.z_dim))
for i in range(self.len_discrete_code):
self.sample_z_[i * self.len_discrete_code] = torch.rand(1, self.z_dim)
for j in range(1, self.len_discrete_code):
self.sample_z_[i * self.len_discrete_code + j] = self.sample_z_[i * self.len_discrete_code]
temp = torch.zeros((self.len_discrete_code, 1))
for i in range(self.len_discrete_code):
temp[i, 0] = i
temp_y = torch.zeros((self.sample_num, 1))
for i in range(self.len_discrete_code):
temp_y[i * self.len_discrete_code: (i + 1) * self.len_discrete_code] = temp
self.sample_y_ = torch.zeros((self.sample_num, self.len_discrete_code)).scatter_(1, temp_y.type(torch.LongTensor), 1)
self.sample_c_ = torch.zeros((self.sample_num, self.len_continuous_code))
# manipulating two continuous code
self.sample_z2_ = torch.rand((1, self.z_dim)).expand(self.sample_num, self.z_dim)
self.sample_y2_ = torch.zeros(self.sample_num, self.len_discrete_code)
self.sample_y2_[:, 0] = 1
temp_c = torch.linspace(-1, 1, 10)
self.sample_c2_ = torch.zeros((self.sample_num, 2))
for i in range(self.len_discrete_code):
for j in range(self.len_discrete_code):
self.sample_c2_[i*self.len_discrete_code+j, 0] = temp_c[i]
self.sample_c2_[i*self.len_discrete_code+j, 1] = temp_c[j]
self.sample_z_, self.sample_y_, self.sample_c_, self.sample_z2_, self.sample_y2_, self.sample_c2_ = \
self.sample_z_.cpu(), self.sample_y_.cpu(), self.sample_c_.cpu(), self.sample_z2_.cpu(), \
self.sample_y2_.cpu(), self.sample_c2_.cpu()
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay)
if args.checkpoint is not None:
try:
load_checkpoint(args.checkpoint, model, optimizer)
print('load checkpoint', args.checkpoint)
except FileNotFoundError:
print('checkpoint not found:', args.checkpoint)
params = list(model.parameters())
params = list(filter(lambda p: p.requires_grad, params))
nparams = sum([np.prod(p.size()) for p in params])
print ('total nubmer of trainable parameters:', nparams)
train_loader, valid_loader, test_loader = dataloader(args.dataset, args.Batchsize, args.cuda)
if not args.train:
model.eval()
data, _ = list(test_loader)[0]
data = data.to(device).view(-1, dim)
#gaussianization(data, model)
reversibility(data, model)
if args.show:
plt.show()
else:
plt.savefig(args.outname, dpi=300, transparent=True)
else:
if args.interactive:
chrono.remove("step_time")
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Train Autoencoder")
parser.add_argument("--valid",
action="store_true",
default=False,
help="Perform validation only.")
args = parser.parse_args()
chrono = Chrono()
progress_bar = ProgressBar()
_, _, trainloader, testloader = dataloader()
autoencoder = get_torch_vars(AutoEncoder(), False)
if torch.cuda.is_available():
autoencoder = torch.nn.DataParallel(autoencoder)
torch.backends.cudnn.benchmark = True
if args.valid:
valid()
exit(0)
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(autoencoder.parameters())
train()
def test_dataloader(self):
x, y = dataloader(mode='train')
self.assertEqual(np.shape(x), (250000, 30))
self.assertEqual(np.shape(y), (250000, ))
@Software: PyCharm
"""
from model import Perceptron
from utils import setup_seed, dataloader
import random
import numpy as np
from sklearn.metrics import accuracy_score
# hyperparameter
iters = 50000
lr = 0.0001
seed = 1234
# main process
setup_seed(seed)
x_train, x_test, y_train, y_test = dataloader(ratio=0.9)
perceptron = Perceptron(feat_dim=30, learning_rate=lr)
train_num = x_train.shape[0]
test_num = x_test.shape[0]
# train
for iter in range(iters): # 每次随机从训练集中选择一个样本进行训练
instance_id = random.randint(0, train_num - 1)
x = x_train[instance_id]
y_true = y_train[instance_id]
y_pred = perceptron.forward(x)
perceptron.backward(x, y_pred=y_pred, y_true=y_true)
print("iteration {} finished. y_true: {}, y_pred: {}.".format(
iter, y_true, y_pred))
# test
def train(self, task_id, traindata, trainlabels):
train_data, train_label, train_attr, no_steps = utils.dataloader(
traindata, trainlabels, traindata, self.args.test_batch_size)
for e in range(self.args.class_epochs):
loss = self.train_epoch(task_id, train_data, train_label, no_steps)
print('epoch:', e + 1, 'class_loss:', loss)
state_dict.update(pretrain_state_dict)
self.model.load_state_dict(state_dict)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--lr', type=float, required=True)
parser.add_argument('--gpu', type=str, default='0')
parser.add_argument('--save-model-path', type=str, required=True)
parser.add_argument('--load-model-path', type=str)
parser.add_argument('--is-train', action='store_true', default=False)
parser.add_argument('--batch-size', type=int, default=20)
parser.add_argument('--data-root', type=str, required=True)
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)
train_loader, eval_loader = dataloader(args.batch_size, args.data_root, '../../data/diagnosis.txt')
model = Model()
model = nn.DataParallel(model.cuda())
if not os.path.exists(args.save_model_path):
os.makedirs(args.save_model_path)
manager = Manager(train_loader, eval_loader, model, args.lr, args.save_model_path)
if args.load_model_path:
manager.load_model_state_dict(args.load_model_path)
if args.is_train:
manager.fit()
def main(args):
## load datasets
train_dataset = dataloader('dogs_cats', 'train')
## split train and validation
num_train = len(train_dataset)
indices = list(range(num_train))
split = 5000
validation_idx = np.random.choice(indices, size=split, replace=False)
train_idx = list(set(indices) - set(validation_idx))
train_sampler = SubsetRandomSampler(train_idx)
validation_sampler = SubsetRandomSampler(validation_idx)
## train and validation loader
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
sampler=train_sampler)
valid_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
sampler=validation_sampler)
## debug
if args.debug:
images, _ = next(iter(train_loader))
grid = torchvision.utils.make_grid(images[:25], nrow=5)
imshow(grid, 'train')
images, _ = next(iter(valid_loader))
grid = torchvision.utils.make_grid(images[:25], nrow=5)
imshow(grid, 'valid')
## define model
model = AutoEncoder()
use_gpu = torch.cuda.is_available()
if use_gpu:
print('cuda is available!')
model.cuda()
## loss and optimizer
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(
model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5)
## log
log_dir = 'logs'
if not os.path.isdir('logs'):
os.mkdir('logs')
## train and valid
best_val = 5
loss_list = []
val_loss_list = []
for epoch in range(args.n_epochs):
loss = train(model, criterion, optimizer, train_loader, use_gpu)
val_loss = valid(model, criterion, valid_loader, use_gpu)
print('epoch {:d}, loss: {:.4f} val_loss: {:.4f}'.format(epoch, loss, val_loss))
if val_loss < best_val:
print('val_loss improved from {:.5f} to {:.5f}!'.format(best_val, val_loss))
best_val = val_loss
model_file = 'epoch{:03d}-{:.3f}.pth'.format(epoch, val_loss)
torch.save(model.state_dict(), os.path.join(log_dir, model_file))
loss_list.append(loss)
val_loss_list.append(val_loss)
def main():
args = parser.parse_args()
if args.exp_name == '':
print(
'ERROR: USE \'--exp-name\' or \'-n\' option to define this experiment\'s name.'
)
sys.exit()
# directories settings
os.makedirs('../logs/outputs', exist_ok=True)
os.makedirs('../logs/models/{}'.format(args.exp_name), exist_ok=True)
OUTPUT_PATH = '../logs/outputs/{}.log'.format(args.exp_name)
MODEL_PATH = '../logs/models/{}/'.format(args.exp_name)
if os.path.exists(OUTPUT_PATH):
print(
'ERROR: This \'--exp-name\' is already used. Use another name for this experiment.'
)
sys.exit()
# recording outputs
sys.stdout = open(OUTPUT_PATH, 'w')
sys.stderr = open(OUTPUT_PATH, 'a')
# tensorboardX
writer = SummaryWriter(log_dir='../logs/tb/{}'.format(args.exp_name))
# cuda settings
args.cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda:0" if args.cuda else "cpu")
print('device: {}'.format(device))
# for fast training
cudnn.benchmark = True
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
# data settings
trainloader, testloader, _ = dataloader(batch_size=args.batch_size)
# Model, Criterion, Optimizer
model = AlexNetCifar10().to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
# print settings
print('=' * 5 + ' settings ' + '=' * 5)
print('TRAINING MODE: {}'.format(args.mode))
if args.mode == 'blur-step':
print('### BLUR CHANGING STEPS ###')
print('Step: 1-10 -> 11-20 -> 21-30 -> 31-40 -> 41-{}'.format(
args.epochs))
print('Sigma: 4 -> 3 -> 2 -> 1 -> none')
# print('Kernel-size: (25, 25) -> (19, 19) -> (13, 13) -> (7, 7) -> none')
print('#' * 20)
elif args.mode == 'blur-half-epochs':
print('### NO BLUR FROM EPOCH {:d} ###'.format(args.epochs // 2))
print('Sigma: {}'.format(args.sigma))
# print('Kernel-size: {}'.format(tuple(args.kernel_size))) # radius = sigma * 3 * 2 + 1
elif args.mode == 'blur-all':
print('Sigma: {}'.format(args.sigma))
# print('Kernel-size: {}'.format(tuple(args.kernel_size))) # radius = sigma * 3 * 2 + 1
if args.blur_val:
print('VALIDATION MODE: blur-val')
print('Random seed: {}'.format(args.seed))
print('Epochs: {}'.format(args.epochs))
print('Learning rate: {}'.format(args.lr))
print('Weight_decay: {}'.format(args.weight_decay))
print()
print(model)
print('=' * 20)
print()
# training
print('Start Training...')
train_time = time.time()
blur = True
for epoch in range(args.start_epoch,
args.epochs): # loop over the dataset multiple times
if args.mode == 'normal':
blur = False
elif args.mode == 'blur-step':
### BLUR-STEP SETTINGS ###
if epoch < 10:
args.sigma = 4
# args.kernel_size = (25, 25) # radius = sigma * 3 * 2 + 1
elif epoch < 20:
args.sigma = 3
# args.kernel_size = (19, 19)
elif epoch < 30:
args.sigma = 2
# args.kernel_size = (13, 13)
elif epoch < 40:
args.sigma = 1
# args.kernel_size = (7, 7)
else:
blur = False
elif args.mode == 'blur-half-epochs':
if epoch >= args.epochs // 2:
blur = False
# ===== train mode =====
train_acc = AverageMeter('train_acc', ':6.2f')
train_loss = AverageMeter('train_loss', ':.4e')
model.train()
for i, data in enumerate(trainloader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data[0], data[1].to(device)
# Blur images
if blur:
if args.mode == 'blur-half-data':
half1, half2 = inputs.chunk(2)
# blur first half images
half1 = GaussianBlurAll(half1, (0, 0), args.sigma)
inputs = torch.cat((half1, half2))
else:
inputs = GaussianBlurAll(inputs, (0, 0), args.sigma)
inputs = inputs.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward + record
outputs = model(inputs)
loss = criterion(outputs, labels)
acc1 = accuracy(outputs, labels, topk=(1, ))
train_loss.update(loss.item(), inputs.size(0))
train_acc.update(acc1[0], inputs.size(0))
# backward + optimize
loss.backward()
optimizer.step()
# record the values in tensorboard
writer.add_scalar('loss/train', train_loss.avg,
epoch + 1) # average loss
writer.add_scalar('acc/train', train_acc.avg, epoch + 1) # average acc
# ===== val mode =====
val_acc = AverageMeter('val_acc', ':6.2f')
val_loss = AverageMeter('val_loss', ':.4e')
model.eval()
with torch.no_grad():
for data in testloader:
inputs, labels = data[0], data[1].to(device)
if args.blur_val:
inputs = GaussianBlurAll(inputs, (0, 0), args.sigma)
inputs = inputs.to(device)
outputs = model(inputs)
loss = criterion(outputs, labels)
acc1 = accuracy(outputs, labels, topk=(1, ))
val_loss.update(loss.item(), inputs.size(0))
val_acc.update(acc1[0], inputs.size(0))
# record the values in tensorboard
writer.add_scalar('loss/val', val_loss.avg, epoch + 1) # average loss
writer.add_scalar('acc/val', val_acc.avg, epoch + 1) # average acc
# ===== save the model =====
if (epoch + 1) % 10 == 0:
save_model(
{
'epoch': epoch + 1,
'arch': 'alexnet-cifar10',
'val_loss': val_loss.avg,
'val_acc': val_acc.avg,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
}, MODEL_PATH, epoch + 1)
print('Finished Training')
print("Training time elapsed: {:.4f}mins".format(
(time.time() - train_time) / 60))
print()
writer.close() # close tensorboardX writer
'''Here, we will see whether we could use PCA
as our regression feature to classifier a stock's class'''
from utils import dataloader, accuracy, confusion_map
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
np.random.seed(5)
# prepare data
X, X_class, X_index, Y, cl2co, class2bbr, bbr2class = dataloader()
X_train, X_test = X[:300], X[300:]
Y_train, Y_test = Y[:300], Y[300:]
# PCA
n = 150
pca = PCA(n_components=n)
pca.fit(X_train)
X_train = pca.transform(X_train)
X_test = pca.transform(X_test)
print('Number of Component:', n)
print('Explain Variance: {:.3f}'.format(sum(pca.explained_variance_ratio_)))
# logistic regression
clf = LogisticRegression(C=60, solver='newton-cg')
clf.fit(X_train, Y_train)
print("WARNING: You have a CUDA device, so you should probably set cuda in params.py to True")
# -----------------------------------------------
"""
In this block
Get train and val data_loader
"""
def loader_param():
img_x = 32
img_y = 128
batch_size = 4
return(img_x, img_y, batch_size)
img_x, img_y, batch_size = loader_param()
dataset = dataset.dataset(image_root="../IAM Dataset/words/", label_root = "../IAM Dataset/ascii/labels.txt", img_x = img_x, img_y = img_y)
train_loader, val_loader = utils.dataloader(dataset = dataset, batch_size = 16, validation_split = 0.2, shuffle_dataset = True)
# def data_loader():
# # train
# train_dataset = dataset.lmdbDataset(root=args.trainroot)
# assert train_dataset
# if not params.random_sample:
# sampler = dataset.randomSequentialSampler(train_dataset, params.batchSize)
# else:
# sampler = None
# train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=params.batchSize, \
# shuffle=True, sampler=sampler, num_workers=int(params.workers), \
# collate_fn=dataset.alignCollate(imgH=params.imgH, imgW=params.imgW, keep_ratio=params.keep_ratio))
# # val
