def main():
args = get_args()
if args.use_dropout == 0:
args.use_dropout = False
if args.use_dropout ==0:
args.use_dropout = False
for x in vars(args).items():
print(x)
#from utils import data_transforms
#print(data_transforms)
if args.lr_sch ==5 and torch.__version__ != '0.4.0' :
print("for cosine annealing, change to torch==0.4.0 in setup.py")
raise AssertionError()
elif args.lr_sch !=5 and torch.__version__ == '0.4.0':
print("warning : this is torch version {}! nsml report will not be recorded".format(torch.__version__))
model, optimizer, scheduler = model_all.get_model(args)
if args.use_gpu:
if torch.cuda.device_count() > 1:
print("[gpu] Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
model = torch.nn.DataParallel(model)
elif torch.cuda.device_count() == 1:
print("[gpu] Let's use", torch.cuda.device_count(), "GPUs!")
else:
print("[gpu] no available gpus")
model = model.cuda()
nsml.bind(infer=infer, model=model, optimizer=optimizer)
if args.pause:
nsml.paused(scope=locals())
nsml.save()
if args.mode == 'train':
dataloaders, dataset_sizes = utils.data_loader(args, train=True, batch_size=args.batch_size)
model = train.train_test(model, optimizer, scheduler, dataloaders, dataset_sizes, args)
utils.save_model(model, 'model_state')
with open('args.pickle', 'wb') as farg:
pickle.dump(args, farg)
loader = utils.data_loader(args, train=False, batch_size=1)
predict, acc = utils.get_forward_result(model, loader, args)
predict = torch.cat(predict, 0)
nsml.bind(save=lambda x: utils.save_csv(x,
data_csv_fname=os.path.join(DATASET_PATH, 'train', 'test') + '/test_data',
results=predict,
test_loader=loader))
nsml.save('result')
def test():
print('Start to test...')
ctx = [mx.gpu(int(i))
for i in args.gpus.split(',')] if args.gpus != '-1' else mx.cpu()
_, test_iter = data_loader(args.batch_size)
model = LeNetPlus()
model.load_params(os.path.join(args.ckpt_dir,
args.prefix + '-best.params'),
ctx=ctx)
start_time = timeit.default_timer()
test_accuracy, features, predictions, labels = evaluate_accuracy(
test_iter, model, ctx)
elapsed_time = timeit.default_timer() - start_time
print("Test_acc: %s, Elapsed_time: %f s" % (test_accuracy, elapsed_time))
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
if args.plotting:
plot_features(features,
labels,
num_classes=args.num_classes,
fpath=os.path.join(args.out_dir, '%s.png' % args.prefix))
def __init__(self, config, model, experiments):
self.config = config
self.model = model
self.experiments = experiments
self.trainloader, _ = data_loader(config)
# checkpoint
self.checkpoint_dir = make_dirs(
os.path.join(self.config.result_path, self.config.checkpoint_path))
self.ckpt = tf.train.Checkpoint(enc=self.model.enc,
dec=self.model.dec,
optim=self.model.optim,
epoch=self.model.global_epoch)
self.ckpt_manager = tf.train.CheckpointManager(
self.ckpt,
directory=self.checkpoint_dir,
checkpoint_name='ckpt',
max_to_keep=2)
# tensorboard
self.tensorboard_dir = make_dirs(
os.path.join(self.config.result_path,
self.config.tensorboard_path))
self.summary_writer = tf.summary.create_file_writer(
self.tensorboard_dir)
def evalidate(flag, args):
with tf.Graph().as_default():
queue_loader = data_loader(False,
batch_size=args.bsize,
num_epochs=args.ep,
dataset_dir=args.dataset_dir)
with slim.arg_scope(resnet_v2.resnet_arg_scope()):
logits, end_points = resnet_v2.resnet_v2_101(queue_loader.images,
num_classes=764,
is_training=flag)
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
"accuracy":
slim.metrics.accuracy(np.argmax(logits, 1), queue_loader.labels)
})
for name, value in names_to_values.items():
op_name = 'eval_{}'.format(name)
op = tf.summary.scalar(op_name, value)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
slim.evaluation.evaluate_once(
master='',
checkpoint_path=args.modelpath,
logdir=args.evallog,
num_evals=queue_loader.num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=slim.get_variables_to_restore())
def __init__(self, config, model):
self.config = config
self.model = model
_, self.testloader = data_loader(config)
# directory to save experimental results
self.dir = make_dirs(os.path.join(self.config.result_path, self.config.experiment_path))
def __init__(self, dataset, data_path="data", task_type="full"):
self.dataset = dataset
self.data_path = data_path
(self.adj, self.train_adj, self.features, self.train_features,
self.labels, self.idx_train, self.idx_val, self.idx_test, self.degree,
self.learning_type) = data_loader(dataset, data_path, "NoNorm", False,
task_type)
#convert some data to torch tensor ---- may be not the best practice here.
self.features = torch.FloatTensor(self.features).float()
self.train_features = torch.FloatTensor(self.train_features).float()
# self.train_adj = self.train_adj.tocsr()
self.labels_torch = torch.LongTensor(self.labels)
self.idx_train_torch = torch.LongTensor(self.idx_train)
self.idx_val_torch = torch.LongTensor(self.idx_val)
self.idx_test_torch = torch.LongTensor(self.idx_test)
# vertex_sampler cache
# where return a tuple
self.pos_train_idx = np.where(self.labels[self.idx_train] == 1)[0]
self.neg_train_idx = np.where(self.labels[self.idx_train] == 0)[0]
# self.pos_train_neighbor_idx = np.where
self.nfeat = self.features.shape[1]
self.nclass = int(self.labels.max().item() + 1)
self.trainadj_cache = {}
self.adj_cache = {}
#print(type(self.train_adj))
self.degree_p = None
def main():
dataset_name = 'K562_USP_UUE'
dataset_path = '/Users/xinzeng/Desktop/research/reproduce/'
global datasets
# input data from preproces
datasets = data_loader(dataset_path + dataset_name+'.h5')
# check whether the saved file exist!
date = '5_4_2020'
save_file_path = dataset_path + date
mkdir(save_file_path)
save_file_path = save_file_path + '/' + dataset_name
mkdir(save_file_path)
best_hyperparameters = input_param(dataset_path + 'K562_USP_UUE_best_model_info')
# saved the hyperparameters of the model
save_f = open(save_file_path + '/best_model_info', 'w')
#run hyperopt to get the best hyperparameters
print(best_hyperparameters)
#run the best parameters on the model
model = best_param_cnn(best_hyperparameters,save_f,dataset_name)
model.save(save_file_path + '/best_model.h5')
save_f.close()
def __init__(self, config, model, experiments):
self.config = config
self.model = model
self.experiments = experiments
self.trainloader, _ = data_loader(config)
# checkpoint
self.checkpoint_dir = make_dirs(
os.path.join(self.config.result_path, self.config.checkpoint_path))
self.ckpt = tf.train.Checkpoint(gen=self.model.gen,
g_optim=self.model.g_optim,
dis=self.model.dis,
d_optim=self.model.d_optim,
epoch=self.model.global_epoch)
self.ckpt_manager = tf.train.CheckpointManager(
self.ckpt,
directory=self.checkpoint_dir,
checkpoint_name='ckpt',
max_to_keep=1)
# tensorboard
self.tensorboard_dir = make_dirs(
os.path.join(self.config.result_path,
self.config.tensorboard_path))
self.summary_writer = tf.summary.create_file_writer(
self.tensorboard_dir)
tf.summary.trace_on(graph=True, profiler=True)
def test_offline():
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
model = Net()
margin = 1.0
threshlod = 1.0
model_path = '/data/chenchao/reid_train/exp11.2/models/epoch_0004-000500_feat.weights'
if not os.path.exists(model_path):
print('huaijinhhhh')
test_list = '../data/val.txt'
image_root = '/home/chenchao/ReID/'
test_loader = data_loader(image_root,
test_list,
shuffle=False,
batch_size=512,
is_visualization=True)
acc = test_vis(model,
test_loader,
model_path,
threshlod,
margin,
is_cuda=True,
is_visualization=True)
print('best_threshold : {:.03f}, best_accuracy:{:.03f}'.format(
threshlod, acc))
def infer_step(sess, model, infer_file, jieba):
iterator = data_loader(infer_file, FLAGS.batch_size, 1, False, "infer")
sess.run(iterator.initializer)
elems = iterator.get_next()
probs = []
while True:
try:
try:
batch_context, batch_utterance, _ = sess.run(elems)
except ValueError:
batch_context, batch_utterance = sess.run(elems)
batch_context = list(
map(lambda d: (' ').join(jieba.lcut(d.decode())),
batch_context))
batch_utterance = list(
map(lambda d: (' ').join(jieba.lcut(d.decode())),
batch_utterance))
batch_context, batch_context_len = get_data(batch_context)
batch_utterance, batch_utterance_len = get_data(batch_utterance)
except tf.errors.OutOfRangeError:
break
feed_dict = {
model.context_embedded: batch_context,
model.utterance_embedded: batch_utterance,
model.context_len: batch_context_len,
model.utterance_len: batch_utterance_len
}
prob = sess.run([model.probs], feed_dict=feed_dict)
prob = list(np.squeeze(prob))
probs += prob
probs = np.mean(probs)
return probs
def debug(model, minibatch=200, device_ids=[0]):
_loader, column_path = data_loader(debug=True,
refresh=True,
minibatch=minibatch,
shuffle=False,
gpu=True,
truncate=True)
fields = field_size(column_path)
embedding_dim = 10
model = model(fields, embedding_dim).cuda()
# model = torch.nn.DataParallel(model, device_ids=device_ids)
loss = torch.nn.NLLLoss().cuda()
lr = 1e-3
solver = torch.optim.Adam(model.parameters(), lr=lr)
for _iter in range(10):
end = default_timer()
_loader.create_iter()
LOSS, correct = 0, 0
_sum = {}
for tensor, labels in _loader._iter:
tensor = tensor.long()
inputs = Variable(tensor)
labels = Variable(labels)
outputs = model(inputs)
_loss = loss(outputs, labels)
LOSS = LOSS + _loss.data[0]
_loss.backward()
solver.step()
pred = outputs.data.max(1)[1]
correct += pred.eq(labels.data).sum()
for x in range(pred.shape[0]):
if pred[x] not in list(_sum.keys()):
_sum[pred[x]] = 1
else:
_sum[pred[x]] = _sum[pred[x]] + 1
break
correct = 100. * correct / _loader._len
LOSS = LOSS / _loader._len
print("loss: {:6f}, accuracy: {:6f}, len: {}, step_time: {:4f}".format(
LOSS, correct, _loader._len,
default_timer() - end))
print("... dict: {}".format(_sum))
def main(preprocessed_dir_path, trained_dir_path, prefix, embedding_dim,
hidden_dim, batch_size):
print(20 * "=", f"Preparing for test", 20 * "=")
if prefix != "":
test_file_name = f"{prefix}_test"
vocab_file_name = f"{prefix}_vocab"
revocab_file_name = f"{prefix}_revocab"
weight_file_name = f"{prefix}_weight"
else:
test_file_name = "test"
vocab_file_name = "vocab"
revocab_file_name = "revocab"
weight_file_name = "weight"
with open(os.path.join(preprocessed_dir_path, vocab_file_name + ".pkl"),
"rb") as fp:
vocab = pickle.load(fp)
with open(os.path.join(preprocessed_dir_path, revocab_file_name + ".pkl"),
"rb") as fp:
revocab = pickle.load(fp)
vocab_size = len(vocab)
padding_idx = vocab[" "]
encoder = Encoder(vocab_size, embedding_dim, hidden_dim,
padding_idx).to(device)
decoder = Decoder(vocab_size, embedding_dim, hidden_dim,
padding_idx).to(device)
encoder.load_state_dict(
torch.load(os.path.join(trained_dir_path,
f"{weight_file_name}.pth"))["encoder"])
decoder.load_state_dict(
torch.load(os.path.join(trained_dir_path,
f"{weight_file_name}.pth"))["decoder"])
expression_ids, answer_ids = data_loader(
os.path.join(preprocessed_dir_path, test_file_name + ".pkl"),
padding_idx)
print(20 * "=", f"Testing", 20 * "=")
predicted_answers = predict(encoder, decoder, vocab, revocab,
expression_ids, batch_size, padding_idx)
answers = []
for ids in answer_ids.detach().numpy().tolist():
answer = int("".join([revocab[str(id)] for id in ids]))
answers.append(answer)
print(20 * "=", f"Calculate Test Result", 20 * "=")
score = 0
missed = []
for i, answer in enumerate(answers):
if predicted_answers[i] == answer:
score += 1
else:
missed.append((answer, predicted_answers[i]))
print(f"Accuracy: {score/len(answers) * 100} ({score}/{len(answers)})")
def __init__(self, dataset, args, data_path="data", task_type="full"):
self.dataset = dataset
self.data_path = data_path
(self.adj, self.train_adj, self.features, self.train_features,
self.labels, self.idx_train, self.idx_val, self.idx_test, self.degree,
self.learning_type) = data_loader(dataset, data_path, "NoNorm", False,
task_type)
#convert some data to torch tensor ---- may be not the best practice here.
self.train_adj = self.adj
self.train_features = self.features
self.learning_type = 'transductive'
self.labels = self.labels.astype(np.int)
if args.pca:
# pca = TruncatedSVD(n_components=600, n_iter=5000, algorithm='arpack')
pca = PCA(n_components=256)
self.features = pca.fit_transform(self.features)
self.train_features = pca.fit_transform(self.train_features)
self.features = torch.FloatTensor(self.features).float()
self.train_features = torch.FloatTensor(self.train_features).float()
# self.train_adj = self.train_adj.tocsr()
if args.train_size and not args.fastmode:
from ssl_utils import get_splits_each_class
self.idx_train, self.idx_val, self.idx_test = get_splits_each_class(
labels=self.labels, train_size=args.train_size)
# print(self.idx_train[:10])
# from ssl_utils import get_few_labeled_splits
# self.idx_train, self.idx_val, self.idx_test = get_few_labeled_splits(
# labels=self.labels, train_size=args.train_size)
if args.fastmode:
from deeprobust.graph.utils import get_train_test
self.idx_train, self.idx_test = get_train_test(
nnodes=self.adj.shape[0],
test_size=1 - args.label_rate,
stratify=self.labels)
self.idx_test = self.idx_test[:1000]
self.labels_torch = torch.LongTensor(self.labels)
self.idx_train_torch = torch.LongTensor(self.idx_train)
self.idx_val_torch = torch.LongTensor(self.idx_val)
self.idx_test_torch = torch.LongTensor(self.idx_test)
# vertex_sampler cache
# where return a tuple
self.pos_train_idx = np.where(self.labels[self.idx_train] == 1)[0]
self.neg_train_idx = np.where(self.labels[self.idx_train] == 0)[0]
# self.pos_train_neighbor_idx = np.where
self.nfeat = self.features.shape[1]
self.nclass = int(self.labels.max().item() + 1)
self.trainadj_cache = {}
self.adj_cache = {}
#print(type(self.train_adj))
self.degree_p = None
def main():
"""Main pipeline."""
# parse command line arguments
pathSet = [
"./myProject/trainData/Mild/", "./myProject/trainData/Aggressive/"
]
for dataPath in pathSet:
# parse argument
args = parse_args(dataPath)
print("Constructing data loaders...")
# load training set and split
trainset, valset = load_data(args.dataroot, args.train_size)
trainloader, validationloader = data_loader(args.dataroot, trainset,
valset, args.batch_size,
args.shuffle,
args.num_workers)
# define model
print("Initialize model ...")
if args.model_name == "default":
model = NetworkNvidia()
elif args.model_name == "modified":
model = ModNet()
# define optimizer and loss function
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
criterion = nn.MSELoss()
# learning rate scheduler
scheduler = MultiStepLR(optimizer, milestones=[30, 50], gamma=0.1)
# resume
if args.resume:
print("Loading a model from checkpoint")
# use pre-trained model
checkpoint = torch.load("model.h5",
map_location=lambda storage, loc: storage)
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
# cuda or cpu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("Selected GPU: ", device)
# training
print("Training Neural Network...")
trainer = Trainer(args.dataroot, args.ckptroot, model, device,
args.epochs, criterion, optimizer, scheduler,
args.start_epoch, trainloader, validationloader)
trainer.train()
def main():
x_test, y_test = data_loader(clean_data_filename)
x_test = data_preprocess(x_test)
bd_model = keras.models.load_model(model_filename)
clean_label_p = np.argmax(bd_model.predict(x_test), axis=1)
class_accu = np.mean(np.equal(clean_label_p, y_test)) * 100
print('Classification accuracy:', class_accu)
def main():
"""Main pipeline."""
# parse command line arguments
args = parse_args()
# load trainig set and split
trainset, valset = load_data(args.dataroot, args.train_size)
print("==> Preparing dataset ...")
trainloader, validationloader = data_loader(args.dataroot, trainset,
valset, args.batch_size,
args.shuffle, args.num_workers)
# define model
print("==> Initialize model ...")
if args.model_name == "nvidia":
model = NetworkNvidia()
elif args.model_name == "light":
model = NetworkLight()
# define optimizer and criterion
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
criterion = nn.MSELoss()
# learning rate scheduler
scheduler = MultiStepLR(optimizer, milestones=[30, 50], gamma=0.1)
# resume
if args.resume:
print("==> Loading checkpoint ...")
# use pre-trained model
checkpoint = torch.load("model.h5",
map_location=lambda storage, loc: storage)
print("==> Loading checkpoint model successfully ...")
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
# cuda or cpu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("==> Use accelerator: ", device)
# training
print("==> Start training ...")
trainer = Trainer(args.ckptroot, model, device, args.epochs, criterion,
optimizer, scheduler, args.start_epoch, trainloader,
validationloader)
trainer.train()
def train_cnn(cnn, cnn_cfg, rl_cfg, game_index_now):
replays_paths = rl_cfg.replay_path
batch_size = cnn_cfg.batch_size
epoch = cnn_cfg.epoch
random_samples, step_size = load_replays(
replays_paths,
batch_size,
game_index_now,
pos_sample_factor=cnn_cfg.pos_sample_factor,
max_N=cnn_cfg.max_N,
valid_game_index_range=cnn_cfg.valid_game_index_range)
cnn_data_loader = data_loader(batch_size, random_samples, step_size)
cnn.train_model(cnn_data_loader, epoch, step_size, save_chkpnt=True)
print("Train CNN done!")
def train(model, training_data, dev_data, learning_rate, batch_size,
max_epoch):
X_train, Y_train = training_data['X'], training_data['Y']
X_dev, Y_dev = dev_data['X'], dev_data['Y']
for i in range(max_epoch):
for X, Y in data_loader(X_train,
Y_train,
batch_size=batch_size,
shuffle=True):
training_loss, grad_Ws, grad_bs = model.compute_gradients(X, Y)
model.update(grad_Ws, grad_bs, learning_rate)
dev_acc = acc(model.predict(X_dev), Y_dev)
print("Epoch {: >3d}/{}\tloss:{:.5f}\tdev_acc:{:.5f}".format(
i + 1, max_epoch, training_loss, dev_acc))
return model
def visualize_label_scan_bottom_right_white_4():
print('loading dataset')
X_test, Y_test = utils.data_loader('%s/%s' % (DATA_DIR, DATA_FILE))
# transform numpy arrays into data generator
test_generator = build_data_loader(X_test, Y_test)
print('loading model')
model_file = '%s/%s' % (MODEL_DIR, MODEL_FILENAME)
model = load_model(model_file)
# initialize visualizer
visualizer = Visualizer(model,
intensity_range=INTENSITY_RANGE,
regularization=REGULARIZATION,
input_shape=INPUT_SHAPE,
init_cost=INIT_COST,
steps=STEPS,
lr=LR,
num_classes=NUM_CLASSES,
mini_batch=MINI_BATCH,
upsample_size=UPSAMPLE_SIZE,
attack_succ_threshold=ATTACK_SUCC_THRESHOLD,
patience=PATIENCE,
cost_multiplier=COST_MULTIPLIER,
img_color=IMG_COLOR,
batch_size=BATCH_SIZE,
verbose=2,
save_last=SAVE_LAST,
early_stop=EARLY_STOP,
early_stop_threshold=EARLY_STOP_THRESHOLD,
early_stop_patience=EARLY_STOP_PATIENCE)
log_mapping = {}
# y_label list to analyze
y_target_list = list(range(NUM_CLASSES))
y_target_list.remove(Y_TARGET)
y_target_list = [Y_TARGET] + y_target_list
for y_target in y_target_list:
print('processing label %d' % y_target)
_, _, logs = visualize_trigger_w_mask(visualizer,
test_generator,
y_target=y_target,
save_pattern_flag=True)
log_mapping[y_target] = logs
def evaluation():
model_path = '/data/chenchao/reid_train/exp11.2/models/epoch_0004-000500_feat.weights'
model = Net()
model = model.cuda()
model.load_full_weights(model_path)
model.eval()
test_list = '../data/val.txt'
image_root = '/home/chenchao/ReID/'
test_loader = data_loader(image_root,
test_list,
shuffle=True,
batch_size=256)
margin = 1.0
#accuracy = test(model, test_loader, 0, margin, threshlod=0.18, log_interval=10)
thre, acc, mean_1, mean_2 = best_test(model, test_loader, model_path)
print('accuracy = {:f}'.format(acc))
def ComputeClassificationAccuracy(images, recons, labels, args, debug=True):
model_paths = {
'CIFAR': 'classifiers/model/cifar-10',
'CelebA': 'classifiers/model/celeba'
}
batch_size = 50
dset = utils.data_loader(images, recons, labels, batch_size)
# normalization, accuracy
sess = tf.Session()
if args.dataset == 'CIFAR':
model = CIFARModel(model_paths[args.dataset],
tiny=False,
mode='eval',
sess=sess)
# TODO: Write CelebA model class
n_data = 0
n_correct_orig = 0
n_correct = 0
total = 0
for images, recons, labels in dset:
total += 1
n_correct_orig += sess.run(model.num_correct,
feed_dict={
model.x_input: images,
model.y_input: labels
})
n_correct += sess.run(model.num_correct,
feed_dict={
model.x_input: recons,
model.y_input: labels
})
n_data += len(images)
acc_orig = float(n_correct_orig) / n_data
acc = float(n_correct) / n_data
print('Original acc: {}'.format(acc_orig))
print('Accuracy: {}'.format(acc))
return acc
def test():
"""
Test model accuracy on test dataset.
测试模型在测试集上的准确率。
"""
print("Start to test...")
ctx = mx.gpu() if args.use_gpu else mx.cpu()
_, test_iter = data_loader(args.batch_size)
model = LeNetPlus()
model.load_parameters(os.path.join(args.ckpt_dir,
args.prefix + "-best.params"),
ctx=ctx,
allow_missing=True)
#
center_net = CenterLoss(num_classes=args.num_classes,
feature_size=args.feature_size,
lmbd=args.lmbd,
ctx=mx.cpu())
center_net.load_parameters(os.path.join(
args.ckpt_dir, args.prefix + "-feature_matrix.params"),
ctx=ctx)
start_time = time.time()
test_accuracy, features, predictions, labels = evaluate_accuracy(
test_iter, model, center_net, args.eval_method, ctx)
elapsed_time = time.time() - start_time
print("Test_acc: %s, Elapsed_time: %f s" % (test_accuracy, elapsed_time))
# make directory
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
# draw feature map
if args.plotting:
plot_features(features,
labels,
num_classes=args.num_classes,
fpath=os.path.join(args.out_dir, "%s.png" % args.prefix))
def train_1pass(model,
training_data,
dev_data,
learning_rate,
batch_size,
print_every=100,
plot_every=10):
X_train, Y_train = training_data['X'], training_data['Y']
X_dev, Y_dev = dev_data['X'], dev_data['Y']
num_samples = 0
print_loss_total = 0
plot_loss_total = 0
plot_losses = []
plot_num_samples = []
for idx, (X, Y) in enumerate(
data_loader(X_train, Y_train, batch_size=batch_size, shuffle=True),
1):
training_loss, grad_Ws, grad_bs = model.compute_gradients(X, Y)
model.update(grad_Ws, grad_bs, learning_rate)
num_samples += Y.shape[1]
print_loss_total += training_loss
plot_loss_total += training_loss
if idx % print_every == 0:
dev_acc = acc(model.predict(X_dev), Y_dev)
print_loss_avg = print_loss_total / print_every
print_loss_total = 0
print("#Samples {: >5d}\tloss:{:.5f}\tdev_acc:{:.5f}".format(
num_samples, print_loss_avg, dev_acc))
if idx % plot_every == 0:
plot_loss_avg = plot_loss_total / plot_every
plot_loss_total = 0
plot_losses.append(plot_loss_avg)
plot_num_samples.append(num_samples)
return model, {"losses": plot_losses, "num_samples": plot_num_samples}
def main(args):
latent_dim = ldim(args.dataset, args.is_realnvp)
model = torch.load(args.generator_path)
model.eval()
print(model)
real_image_num = 1
reconstruct_image_num = 1
_, test_loader = data_loader(args)
if not os.path.exists(args.output_dir): os.makedirs(args.output_dir)
for batch_idx, (data, target) in enumerate(test_loader):
if data.size()[0] != args.batch_size:
continue
data, target = torch.tensor(data).to(
args.device), torch.tensor(target).to(args.device)
data = (data + 1.0) / 2.0
z = model(data, reverse=False)
z = z.to(args.device)
out, _ = model(z, reverse=True)
out = out.to('cpu').data.numpy()
out = (out + 1.0) / 2.0
out = np.clip(out, 0.0, 1.0)
for sample in out:
out_file = os.path.join(args.output_dir,
f"{reconstruct_image_num}reconstruct.png")
if sample.shape[0] == 1: sample = sample[0]
scmisc.toimage(sample, cmin=0.0, cmax=1.0).save(out_file)
reconstruct_image_num += 1
data = data.to('cpu').data.numpy()
data = (data + 1.0) / 2.0
data = np.clip(data, 0.0, 1.0)
for sample in data:
out_file = os.path.join(args.output_dir,
f"{real_image_num}real.png")
if sample.shape[0] == 1: sample = sample[0]
scmisc.toimage(sample, cmin=0.0, cmax=1.0).save(out_file)
real_image_num += 1
def test_step(sess,
model,
valid_file,
step=None,
summary_op=None,
summary_writer=None):
iterator = data_loader(valid_file, FLAGS.batch_size, 1)
sess.run(iterator.initializer)
losses, iters = 0, 0
elems = iterator.get_next()
while True:
try:
batch_context, batch_utterance, batch_labels = sess.run(elems)
batch_context, batch_context_len = get_data(batch_context)
batch_utterance, batch_utterance_len = get_data(batch_utterance)
batch_labels = batch_labels.reshape(-1, 1)
except tf.errors.OutOfRangeError:
break
feed_dict = {
model.context_embedded: batch_context,
model.utterance_embedded: batch_utterance,
model.context_len: batch_context_len,
model.utterance_len: batch_utterance_len,
model.labels: batch_labels
}
if summary_op is not None:
summaries, loss = sess.run([summary_op, model.loss],
feed_dict=feed_dict)
summary_writer.add_summary(summaries, step)
else:
loss = sess.run([model.loss], feed_dict=feed_dict)[0]
losses += loss
iters += 1
return losses / iters
def evaluation(model_path, test_list):
model = Net()
model = model.cuda()
model.load_full_weights(model_path)
model.eval()
image_root = '/path/to/ReId_data/image/'
if False:
train_list = 'data/debug.txt'
test_list = '../data/test_debug.txt'
else:
train_list = 'data/train_all_5set.txt'
test_list = '../data/test_all_5set.txt'
test_loader = data_loader(image_root,
test_list,
shuffle=True,
batch_size=32)
margin = 1
accuracy = test(model,
test_loader,
0,
margin,
threshlod=10,
log_interval=10)
def main():
torch.cuda.manual_seed(seed)
cudnn.benchmark = CUDNN
# model
model = tiramisu.FcDnSubtle(in_channels=8, n_classes=N_CLASSES)
model = model.cuda()
print(' + Number of params: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
model.apply(utils.weights_init)
optimizer = optim.RMSprop(model.parameters(),
lr=LEARNING_RATE,
weight_decay=WEIGHT_DECAY,
eps=1e-12)
criterion = nn.NLLLoss2d().cuda()
exp = experiment.Experiment(EXPNAME, EXPERIMENT)
exp.init()
START_EPOCH = exp.epoch
END_EPOCH = START_EPOCH + N_EPOCHS
# for epoch in range(1):
for epoch in range(START_EPOCH, END_EPOCH):
since = time.time()
### Collect data ###
# delete existing folder and old data
if os.path.exists(res_root_path):
shutil.rmtree(res_root_path)
utils.collect_data(ori_train_base_rp, res_train_base_rp)
utils.collect_data(ori_val_base_rp, res_val_base_rp)
# data loader
train_loader, val_loader = utils.data_loader(res_root_path)
### Train ###
trn_loss, trn_err = utils.train(model, train_loader, optimizer,
criterion, epoch)
print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format(
epoch, trn_loss, trn_err))
time_elapsed = time.time() - since
print('Train Time {:.0f}m {:.0f}s'.format(time_elapsed // 60,
time_elapsed % 60))
### val ###
val_loss, val_err = utils.test(model, val_loader, criterion, epoch)
print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err))
time_elapsed = time.time() - since
print('Total Time {:.0f}m {:.0f}s\n'.format(time_elapsed // 60,
time_elapsed % 60))
### Save Metrics ###
exp.save_history('train', trn_loss, trn_err)
exp.save_history('val', val_loss, val_err)
### Checkpoint ###
exp.save_weights(model, trn_loss, val_loss, trn_err, val_err)
exp.save_optimizer(optimizer, val_loss)
## Early Stopping ##
if (epoch - exp.best_val_loss_epoch) > MAX_PATIENCE:
print(("Early stopping at epoch %d since no " +
"better loss found since epoch %.3").format(
epoch, exp.best_val_loss))
break
utils.adjust_learning_rate(LEARNING_RATE, LR_DECAY, optimizer, epoch,
DECAY_LR_EVERY_N_EPOCHS)
exp.epoch += 1
ind = sio.loadmat(args.area+'/cva_ref.mat')
cva_ind = ind['cva_ref']
cva_ind = np.reshape(cva_ind, newshape=[-1])
i1, i2 = utils.getTrainSamples(cva_ind, im1, im2, args.trn)
loss_log, vpro, fcx, fcy, bval = dsfa(
xtrain=i1, ytrain=i2, xtest=im1, ytest=im2, net_shape=net_shape, args=args)
imm, magnitude, differ_map = utils.linear_sfa(fcx, fcy, vpro, shape=img_shape)
magnitude = np.reshape(magnitude, img_shape[0:-1])
differ = differ_map
change_map = np.reshape(utils.kmeans(np.reshape(magnitude, [-1])), img_shape[0:-1])
# magnitude
acc_un, acc_chg, acc_all2, acc_tp = utils.metric(1-change_map, chg_map)
acc_un, acc_chg, acc_all3, acc_tp = utils.metric(change_map, chg_map)
plt.imsave('results.png',change_map, cmap='gray')
#plt.show()
return None
if __name__ == '__main__':
args = parser()
img1, img2, chg_map = utils.data_loader(area=args.area)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
main(img1, img2, chg_map, args=args)
from numpy.random import seed
import tensorflow as tf
from tensorflow import set_random_seed
from args import Parser
from utils import data_loader
from Seq2Cluster_test import Seq2Cluster
seed(1)
set_random_seed(2)
opts = Parser().parse()
os.environ["CUDA_VISIBLE_DEVICES"] = opts['gpu']
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
dataloader = data_loader(opts['datapath'], opts['dataset'], opts['split'])
tf.reset_default_graph()
model = Seq2Cluster(opts)
if opts['test'] is False:
model.train(dataloader['train_segdata'], dataloader['test_labels'],
dataloader['test_segdata'], dataloader['train_segnum_list'],
dataloader['test_segnum_list'])
else:
model.test(dataloader['test_segdata'], dataloader['test_labels'],
dataloader['test_segnum_list'])
def main(_):
VERBOSE_ = 1
parser = OptionParser()
parser.add_option("--verbose",
action="store_const",
const=1,
dest="verbose",
help="verbose mode")
parser.add_option("-t",
"--train",
dest="train_path",
help="train file path",
metavar="train_path")
parser.add_option("-v",
"--validation",
dest="validation_path",
help="validation file path",
metavar="validation_path")
parser.add_option("-m",
"--model",
dest="model_dir",
help="dir path to save model",
metavar="model_dir")
parser.add_option("-i",
"--iters",
dest="training_iters",
help="training iterations",
metavar="training_iters")
(options, args) = parser.parse_args()
if options.verbose == 0: VERBOSE_ = 0
train_path = options.train_path
if train_path == None:
parser.print_help()
exit(1)
validation_path = options.validation_path
model_dir = options.model_dir
if model_dir == None:
parser.print_help()
exit(1)
if not os.path.isdir(model_dir):
os.makedirs(model_dir)
training_iters = options.training_iters
if not training_iters: training_iters = 30
training_iters = int(training_iters)
config = config_()
config.max_max_epoch = training_iters
eval_config = config_()
path = 'data/char2vec_50.model'
sys.stderr.write("embedding:%s\n" % (path))
cmodel = CharEmbedding(path)
global index2word
index2word = cmodel.index_2_word()
train_data, valid_data, vocab_s, max_len = data_loader(train_path, cmodel)
vocab_s = cmodel.vocab_size()
eval_config.batch_size = 1
config.vocab_size = vocab_s
eval_config.vocab_size = vocab_s
config.num_steps = max_len
eval_config.num_steps = max_len
# print train_data
NUM_THREADS = 5
cf = tf.ConfigProto(intra_op_parallelism_threads=NUM_THREADS,
inter_op_parallelism_threads=NUM_THREADS,
log_device_placement=False)
with tf.Graph().as_default(), tf.Session() as session:
stddd = 0.1
initializer = tf.truncated_normal_initializer(config.init_scale,
stddev=stddd)
sys.stderr.write("stddev:%f\n" % (stddd))
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = CWSModel(is_training=True, config=config)
sys.stderr.write(
"Vocab Size: %d\nTraining Samples: %d\nValid Samples %d\n Layers:%d\n"
% (vocab_s, len(train_data[0]), len(
valid_data[0]), config.num_layers))
sys.stderr.write(
"Hidden Size: %d\nEmbedding size: %d\nWindow Size %d\nNorm %d\n" %
(config.hidden_size, config.embedding_size, config.num_input,
config.max_grad_norm))
tf.initialize_all_variables().run()
m.load_embedding(session, cmodel.embedding_matrix())
saver = tf.train.Saver() # save all variables
for i in range(config.max_max_epoch):
m.is_training = True
m.config.batch_size = 20
train_accuracy = run_epoch(session,
m,
train_data,
m.train_op,
cmodel,
verbose=True)
sys.stderr.write("Epoch: %d Train accuracy: %.3f\n" %
(i + 1, 1 - train_accuracy))
m.is_training = False
m.config.batch_size = 20
valid_accuracy = run_epoch(session,
m,
valid_data,
tf.no_op(),
cmodel,
verbose=False)
sys.stderr.write("Epoch: %d Valid accuracy: %.3f\n" %
(i + 1, 1 - valid_accuracy))
sys.stderr.write('save model(%d)\n' % (i))
saver.save(session, model_dir + '/' + 'model_' + str(i) + '.ckpt')
