def train_model():
data_path = get_config("data-file")
batchs = get_config("batchs")
half_batch = batchs // 2
quarter_batch = half_batch // 2
n_batchs = (get_config("train-datasets")
or train_set.num_examples) // batchs
feature_dim = get_config("feature-dim") or 40
noise_dim = get_config("noise-dim") or 10
input_dim = feature_dim + noise_dim
train_set = H5PYDataset(data_path, which_sets=('train', ))
handle = train_set.open()
generator, discriminator, gan = build_net(input_dim)
save_result(0, generator, np.zeros(shape=(1, feature_dim)))
for i in range(get_config("epochs")):
for j in range(n_batchs):
imgs, features = get_batch(train_set, handle, j * batchs)
idx = np.random.randint(0, imgs.shape[0], half_batch)
real_imgs = imgs[idx]
real_features = features[idx]
gen_features = get_features(features)[np.random.randint(
0, imgs.shape[0], half_batch)]
noise = np.random.normal(0, 1, (half_batch, noise_dim))
gen_imgs = generator.predict([noise, gen_features])
# real feature and real img
d_loss_real = discriminator.train_on_batch(
[real_imgs], [np.ones((half_batch, 1)), real_features])
# fake feature and fake img
d_loss_fake = discriminator.train_on_batch(
[gen_imgs], [np.zeros((half_batch, 1)), gen_features])
d_loss = np.add(d_loss_real, d_loss_fake) * 0.5
# train Generator
noise = np.random.normal(0, 1, (batchs, noise_dim))
gen_features = get_features(features)
g_loss = gan.train_on_batch([noise, gen_features],
[np.ones((batchs, 1)), gen_features])
print(
"%d [D loss: %f, acc.: %.2f%%] [G loss: %f] %f%%" %
(i, d_loss[0], 100 * d_loss[1], g_loss[0], j * 100 / n_batchs))
if i % 10 == 0 and get_config("env") == "GPU":
save_result(i, generator, gen_features[0:1, :feature_dim])
#!/usr/bin/env python3
import tensorflow as tf
from drive import Simulator, PIController
import model
import config
from utils import *
if __name__ == "__main__":
X, keep_prob, pred = model.build_net(trainable=False)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
cp = tf.train.get_checkpoint_state(config.SAVE_FOLDER)
if cp and cp.model_checkpoint_path:
saver.restore(sess, cp.model_checkpoint_path)
print("Checkpoint Loaded", cp.model_checkpoint_path)
@static_vars(speed_controller=PIController())
def controller(data):
steering = sess.run(pred,
feed_dict = { X: [model.preprocess(base64_to_cvmat(data["image"]), config.INPUT_IMAGE_CROP)],
keep_prob: 1.0})[0]
throttle = controller.speed_controller.update(float(data["speed"]))
return steering, throttle
sim = Simulator(controller)
sim.init()
sim.run()
def run_eval():
with tf.Graph().as_default(), tf.device('/cpu:0'):
### Read images and labels in from tfrecords file.
image_one, label_one = captcha.inputs(False, FLAGS.batch_size, "valid_one")
image_two, label_two = captcha.inputs(False, FLAGS.batch_size, "valid_two")
### Change the shape from
### [ img 1 : x x x x x x x x ]
### [ img 2 : x x x x x x x x ]
### [ ... ...]
### [ img n : x x x x x x x x ]
### to
### [ img 1 img 2 img 3 ... img n]
### [ x x x x ]
### [ ... ... ... ...]
### [ x x x x ]
image_one = tf.reshape(image_one, [-1, IMAGE_HEIGHT * IMAGE_WIDTH])
image_two = tf.reshape(image_two, [-1, IMAGE_HEIGHT * IMAGE_WIDTH])
image_one = tf.transpose(image_one)
image_two = tf.transpose(image_two)
label_one = tf.reshape(label_one, [-1, 10])
label_two = tf.reshape(label_two, [-1, 10])
label_one = tf.transpose(label_one)
label_two = tf.transpose(label_two)
### Build Net
### logits_one and logits_two are the outputs before softmax.
logits_one, logits_two = captcha.build_net(image_one, image_two)
### Use softmax and cross-entropy to calculate loss.
loss_one, loss_two = captcha.loss(logits_one, logits_two, label_one, label_two)
### Evaluate
eval_correct_one = captcha.correct_num(logits_one, label_one)
eval_correct_two = captcha.correct_num(logits_two, label_two)
sess = tf.Session()
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(TRAIN_DIR_ONE))
saver.restore(sess, tf.train.latest_checkpoint(TRAIN_DIR_TWO))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
num_iter = int(math.ceil(FLAGS.num_examples / FLAGS.batch_size))
true_count_one = 0
true_count_two = 0
total_true_count_one = 0
total_true_count_two = 0
total_sample_count = num_iter * FLAGS.batch_size
step = 0
print('>> loop: %d, total_sample_count: %d' % (num_iter, total_sample_count))
while step < num_iter and not coord.should_stop():
true_count_one = sess.run(eval_correct_one)
true_count_two = sess.run(eval_correct_two)
total_true_count_one += true_count_one
total_true_count_two += true_count_two
precision_one = true_count_one / FLAGS.batch_size
precision_two = true_count_two / FLAGS.batch_size
print('>> %s Step %d Net 1: true/total: %d/%d precision @ 1 = %.3f'
%(datetime.now(), step, true_count_one, FLAGS.batch_size, precision_one))
print('>> %s Step %d Net 2: true/total: %d/%d precision @ 1 = %.3f'
%(datetime.now(), step, true_count_two, FLAGS.batch_size, precision_two))
step += 1
precision_one = total_true_count_one / total_sample_count
precision_two = total_true_count_two / total_sample_count
print('>> %s Net 1 true/total: %d/%d precision @ 1 = %.3f'
%(datetime.now(), total_true_count_one, total_sample_count, precision_one))
print('>> %s Net 2 true/total: %d/%d precision @ 1 = %.3f'
%(datetime.now(), total_true_count_two, total_sample_count, precision_two))
except Exception as e:
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
sess.close()
pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL)
print_info('===> Evaluating detections', ['yellow', 'bold'])
testset.evaluate_detections(all_boxes, save_folder)
print_info('Detect time per image: {:.3f}s'.format(tot_detect_time /
(num_images - 1)))
print_info('Nms time per image: {:.3f}s'.format(tot_nms_time /
(num_images - 1)))
print_info('Total time per image: {:.3f}s'.format(
(tot_detect_time + tot_nms_time) / (num_images - 1)))
print_info('FPS: {:.3f} fps'.format(
(num_images - 1) / (tot_detect_time + tot_nms_time)))
if __name__ == '__main__':
net = build_net('test',
size=cfg.model.input_size,
config=cfg.model.m2det_config)
init_net(net, cfg, args.trained_model)
print_info('===> Finished constructing and loading model',
['yellow', 'bold'])
net.eval()
_set = 'eval_sets' if not args.test else 'test_sets'
testset = get_dataloader(cfg, args.dataset, _set)
if cfg.test_cfg.cuda:
net = net.cuda()
cudnn.benchmark = True
else:
net = net.cpu()
detector = Detect(cfg.model.m2det_config.num_classes, cfg.loss.bkg_label,
anchor_config)
save_folder = os.path.join(cfg.test_cfg.save_folder, args.dataset)
def run_train():
"""Train CAPTCHA for a number of steps."""
with tf.Graph().as_default():
### Read images and labels in from tfrecords file.
image_one, label_one = captcha.inputs(True, FLAGS.batch_size,
"train_one")
image_two, label_two = captcha.inputs(True, FLAGS.batch_size,
"train_two")
### Change the shape from
### [ img 1 : x x x x x x x x ]
### [ img 2 : x x x x x x x x ]
### [ ... ...]
### [ img n : x x x x x x x x ]
### to
### [ img 1 img 2 img 3 ... img n]
### [ x x x x ]
### [ ... ... ... ...]
### [ x x x x ]
image_one = tf.reshape(image_one, [-1, IMAGE_HEIGHT * IMAGE_WIDTH])
image_two = tf.reshape(image_two, [-1, IMAGE_HEIGHT * IMAGE_WIDTH])
image_one = tf.transpose(image_one)
image_two = tf.transpose(image_two)
label_one = tf.reshape(label_one, [-1, 10])
label_two = tf.reshape(label_two, [-1, 10])
label_one = tf.transpose(label_one)
label_two = tf.transpose(label_two)
### Build Net
### logits_one and logits_two are the outputs before softmax.
logits_one, logits_two = captcha.build_net(
image_one,
image_two,
dropout_rate=DROPOUT_RATE,
regularization_rate=REGULARIZATION_RATE)
### Use softmax and cross-entropy to calculate loss.
reg_loss_1, reg_loss_2 = captcha.reg_loss()
loss_one, loss_two = captcha.loss(logits_one, logits_two, label_one,
label_two)
eval_one = captcha.correct_rate(logits_one, label_one)
eval_two = captcha.correct_rate(logits_two, label_two)
tf.summary.scalar('loss_one', loss_one)
tf.summary.scalar('loss_two', loss_two)
tf.summary.scalar('eval_one', eval_one)
tf.summary.scalar('eval_two', eval_two)
sum_merged = tf.summary.merge_all()
### Attach the optimizers to two nets.
train_op_one, train_op_two, train_op_both = captcha.training(
loss_one, loss_two)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess = tf.Session()
sess.run(init_op)
#eval_one = sess.run([captcha.evaluation(logits_one, label_one)])
#eval_two = sess.run([captcha.evaluation(logits_two, label_two)])
#print('>> Step %d run_train: accr_one = %.2f, accr_two = %.2f (%.3f sec)' % (step, eval_one,
# eval_two, duration))
### visualize the compute graph
train_summary_writer = tf.summary.FileWriter(VISUAL_DIR,
tf.get_default_graph())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
step = 0
while not coord.should_stop():
start_time = time.time()
### Run a batch to train.
### Save a runtime_metadata after 1000 batches.
summary_scl = None
### Different train method
if TRAIN_METHOD == "M1":
if step % 1000 == 0:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_meta = tf.RunMetadata()
summary_scl, _, loss_value_one = sess.run(
[sum_merged, train_op_one, loss_one],
options=run_options,
run_metadata=run_meta)
summary_scl, _, loss_value_two = sess.run(
[sum_merged, train_op_two, loss_two],
options=run_options,
run_metadata=run_meta)
train_summary_writer.add_run_metadata(run_meta,
'step%06d' %
step,
global_step=step)
else:
summary_scl, _, loss_value_one = sess.run(
[sum_merged, train_op_one, loss_one])
summary_scl, _, loss_value_two = sess.run(
[sum_merged, train_op_two, loss_two])
elif TRAIN_METHOD == "M2":
summary_scl, _, loss_value_one, loss_value_two = sess.run(
[sum_merged, train_op_both, loss_one, loss_two])
duration = time.time() - start_time
### Output the status after 100 batchs.
if step % 100 == 0:
print(
'>> Step %d run_train: loss_one = %.2f, loss_two = %.2f (%.3f sec)'
% (step, loss_value_one, loss_value_two, duration))
reg_loss_value_1, reg_loss_value_2 = sess.run(
[reg_loss_1, reg_loss_2])
print(
'>> Step %d run_train: reg_loss_1 = %.2f, reg_loss_2 = %.2f (%.3f sec)'
% (step, reg_loss_value_1, reg_loss_value_2, duration))
summary_scl, eval_value_one = sess.run(
[sum_merged, eval_one])
summary_scl, eval_value_two = sess.run(
[sum_merged, eval_two])
train_summary_writer.add_summary(summary_scl,
global_step=step)
print(
'>> Step %d run_train: accr_one = %.2f, accr_two = %.2f (%.3f sec)'
% (step, eval_value_one, eval_value_two, duration))
### Save a checkpoint after 5000 batchs.
if step % 5000 == 0:
print('>> %s Saving in %s' %
(datetime.now(), CHECKPOINT_ONE))
print('>> %s Saving in %s' %
(datetime.now(), CHECKPOINT_TWO))
saver.save(sess, CHECKPOINT_ONE, global_step=step)
saver.save(sess, CHECKPOINT_TWO, global_step=step)
if step == 60000:
coord.request_stop()
coord.join(threads)
sess.close()
return 0
step += 1
except Exception as e:
print('>> %s Saving in %s' % (datetime.now(), CHECKPOINT_ONE))
print('>> %s Saving in %s' % (datetime.now(), CHECKPOINT_TWO))
saver.save(sess, CHECKPOINT_ONE, global_step=step)
saver.save(sess, CHECKPOINT_TWO, global_step=step)
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
train_summary_writer.close()
sess.close()
ds = ds.prefetch(buffer_size=batch_size * 16)
iterator = tf.data.Iterator.from_structure(ds.output_types, ds.output_shapes)
images, labels = iterator.get_next()
training_init_op = iterator.make_initializer(ds)
# In[5]:
image_prep_fn = preprocessing_factory.get_preprocessing('inception_v1',
is_training=False)
images_preped = image_prep_fn(images, None, None)
print images, images_preped
import model
class_logits = model.build_net(images_preped, num_classes, True, args.model)
labels_oh = tf.one_hot(labels,
num_classes,
on_value=1.,
off_value=0.,
dtype=tf.float32)
cls_loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_oh,
logits=class_logits)
cls_loss = tf.reduce_mean(cls_loss)
reg_loss = 0. #tf.add_n(tf.losses.get_regularization_losses())
tot_loss = cls_loss + reg_loss
def main():
global net
global test_loader
global scatter
parser = argparse.ArgumentParser()
# generic params
parser.add_argument(
"--name",
default=datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
help="Name to store the log file as",
)
parser.add_argument("--resume", help="Path to log file to resume from")
parser.add_argument("--encoder", default="FSEncoder", help="Encoder")
parser.add_argument("--decoder", default="DSPN", help="Decoder")
parser.add_argument("--epochs",
type=int,
default=10,
help="Number of epochs to train with")
parser.add_argument("--latent",
type=int,
default=32,
help="Dimensionality of latent space")
parser.add_argument("--dim",
type=int,
default=64,
help="Dimensionality of hidden layers")
parser.add_argument("--lr",
type=float,
default=1e-2,
help="Outer learning rate of model")
parser.add_argument("--batch-size",
type=int,
default=32,
help="Batch size to train with")
parser.add_argument("--num-workers",
type=int,
default=4,
help="Number of threads for data loader")
parser.add_argument(
"--dataset",
choices=["mnist", "clevr-box", "clevr-state"],
help="Use MNIST dataset",
)
parser.add_argument(
"--no-cuda",
action="store_true",
help="Run on CPU instead of GPU (not recommended)",
)
parser.add_argument("--train-only",
action="store_true",
help="Only run training, no evaluation")
parser.add_argument("--eval-only",
action="store_true",
help="Only run evaluation, no training")
parser.add_argument("--multi-gpu",
action="store_true",
help="Use multiple GPUs")
parser.add_argument("--show",
action="store_true",
help="Plot generated samples in Tensorboard")
parser.add_argument("--supervised", action="store_true", help="")
parser.add_argument("--baseline",
action="store_true",
help="Use baseline model")
parser.add_argument("--export-dir",
type=str,
help="Directory to output samples to")
parser.add_argument("--export-n",
type=int,
default=10**9,
help="How many samples to output")
parser.add_argument(
"--export-progress",
action="store_true",
help="Output intermediate set predictions for DSPN?",
)
parser.add_argument(
"--full-eval",
action="store_true",
help=
"Use full evaluation set (default: 1/10 of evaluation data)", # don't need full evaluation when training to save some time
)
parser.add_argument(
"--mask-feature",
action="store_true",
help="Treat mask as a feature to compute loss with",
)
parser.add_argument(
"--inner-lr",
type=float,
default=800,
help="Learning rate of DSPN inner optimisation",
)
parser.add_argument(
"--iters",
type=int,
default=10,
help="How many DSPN inner optimisation iteration to take",
)
parser.add_argument(
"--huber-repr",
type=float,
default=1,
help="Scaling of representation loss term for DSPN supervised learning",
)
parser.add_argument(
"--loss",
choices=["hungarian", "chamfer"],
default="hungarian",
help="Type of loss used",
)
args = parser.parse_args()
train_writer = SummaryWriter(f"runs/{args.name}", purge_step=0)
net = model.build_net(args)
if not args.no_cuda:
net = net.cuda()
if args.multi_gpu:
net = torch.nn.DataParallel(net)
optimizer = torch.optim.Adam(
[p for p in net.parameters() if p.requires_grad], lr=args.lr)
if args.dataset == "mnist":
dataset_train = data.MNISTSet(train=True, full=args.full_eval)
dataset_test = data.MNISTSet(train=False, full=args.full_eval)
else:
dataset_train = data.CLEVR("clevr",
"train",
box=args.dataset == "clevr-box",
full=args.full_eval)
dataset_test = data.CLEVR("clevr",
"val",
box=args.dataset == "clevr-box",
full=args.full_eval)
if not args.eval_only:
train_loader = data.get_loader(dataset_train,
batch_size=args.batch_size,
num_workers=args.num_workers)
if not args.train_only:
test_loader = data.get_loader(
dataset_test,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False,
)
tracker = track.Tracker(
train_mae=track.ExpMean(),
train_last=track.ExpMean(),
train_loss=track.ExpMean(),
test_mae=track.Mean(),
test_last=track.Mean(),
test_loss=track.Mean(),
)
if args.resume:
log = torch.load(args.resume)
weights = log["weights"]
n = net
if args.multi_gpu:
n = n.module
n.load_state_dict(weights, strict=True)
def run(net, loader, optimizer, train=False, epoch=0, pool=None):
writer = train_writer
if train:
net.train()
prefix = "train"
torch.set_grad_enabled(True)
else:
net.eval()
prefix = "test"
torch.set_grad_enabled(False)
total_train_steps = args.epochs * len(loader)
if args.export_dir:
true_export = []
pred_export = []
iters_per_epoch = len(loader)
loader = tqdm(
loader,
ncols=0,
desc="{1} E{0:02d}".format(epoch, "train" if train else "test "),
)
for i, sample in enumerate(loader, start=epoch * iters_per_epoch):
# input is either a set or an image
input, target_set, target_mask = map(lambda x: x.cuda(), sample)
# forward evaluation through the network
(progress, masks, evals,
gradn), (y_enc, y_label) = net(input, target_set, target_mask)
progress_only = progress
# if using mask as feature, concat mask feature into progress
if args.mask_feature:
target_set = torch.cat(
[target_set, target_mask.unsqueeze(dim=1)], dim=1)
progress = [
torch.cat([p, m.unsqueeze(dim=1)], dim=1)
for p, m in zip(progress, masks)
]
if args.loss == "chamfer":
# dim 0 is over the inner iteration steps
# target set is broadcasted over dim 0
set_loss = utils.chamfer_loss(torch.stack(progress),
target_set.unsqueeze(0))
else:
# dim 0 is over the inner iteration steps
a = torch.stack(progress)
# target set is explicitly broadcasted over dim 0
b = target_set.repeat(a.size(0), 1, 1, 1)
# flatten inner iteration dim and batch dim
a = a.view(-1, a.size(2), a.size(3))
b = b.view(-1, b.size(2), b.size(3))
set_loss = utils.hungarian_loss(progress[-1],
target_set,
thread_pool=pool).unsqueeze(0)
# Only use representation loss with DSPN and when doing general supervised prediction, not when auto-encoding
if args.supervised and not args.baseline:
repr_loss = args.huber_repr * F.smooth_l1_loss(y_enc, y_label)
loss = set_loss.mean() + repr_loss.mean()
else:
loss = set_loss.mean()
# restore progress variable to not contain masks for correct exporting
progress = progress_only
# Outer optim step
if train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Tensorboard tracking of metrics for debugging
tracked_last = tracker.update("{}_last".format(prefix),
set_loss[-1].item())
tracked_loss = tracker.update("{}_loss".format(prefix),
loss.item())
if train:
writer.add_scalar("metric/set-loss",
loss.item(),
global_step=i)
writer.add_scalar("metric/set-last",
set_loss[-1].mean().item(),
global_step=i)
if not args.baseline:
writer.add_scalar("metric/eval-first",
evals[0].mean().item(),
global_step=i)
writer.add_scalar("metric/eval-last",
evals[-1].mean().item(),
global_step=i)
writer.add_scalar(
"metric/max-inner-grad-norm",
max(g.item() for g in gradn),
global_step=i,
)
writer.add_scalar(
"metric/mean-inner-grad-norm",
sum(g.item() for g in gradn) / len(gradn),
global_step=i,
)
if args.supervised:
writer.add_scalar("metric/repr_loss",
repr_loss.item(),
global_step=i)
# Print current progress to progress bar
fmt = "{:.6f}".format
loader.set_postfix(
last=fmt(tracked_last),
loss=fmt(tracked_loss),
bad=fmt(evals[-1].detach().cpu().item() *
1000) if not args.baseline else 0,
)
# Store predictions to be exported
if args.export_dir:
if len(true_export) < args.export_n:
for p, m in zip(target_set, target_mask):
true_export.append(p.detach().cpu())
progress_steps = []
for pro, mas in zip(progress, masks):
# pro and mas are one step of the inner optim
# score boxes contains the list of predicted elements for one step
score_boxes = []
for p, m in zip(pro.cpu().detach(),
mas.cpu().detach()):
score_box = torch.cat([m.unsqueeze(0), p], dim=0)
score_boxes.append(score_box)
progress_steps.append(score_boxes)
for b in zip(*progress_steps):
pred_export.append(b)
# Plot predictions in Tensorboard
if args.show and not train:
name = f"set/epoch-{epoch}/img-{i}"
# thresholded set
progress.append(progress[-1])
masks.append((masks[-1] > 0.5).float())
# target set
if args.mask_feature:
# target set is augmented with masks, so remove them
progress.append(target_set[:, :-1])
else:
progress.append(target_set)
masks.append(target_mask)
# intermediate sets
for j, (s, ms) in enumerate(zip(progress, masks)):
if args.dataset == "clevr-state":
continue
s, ms = utils.scatter_masked(
s,
ms,
binned=args.dataset.startswith("clevr"),
threshold=0.5
if args.dataset.startswith("clevr") else None,
)
tag_name = f"{name}" if j != len(
progress) - 1 else f"{name}-target"
if args.dataset == "clevr-box":
img = input[0].detach().cpu()
writer.add_image_with_boxes(tag_name,
img,
s.transpose(0, 1),
global_step=j)
elif args.dataset == "clevr-state":
pass
else: # mnist
fig = plt.figure()
y, x = s
y = 1 - y
ms = ms.numpy()
rgba_colors = np.zeros((ms.size, 4))
rgba_colors[:, 2] = 1.0
rgba_colors[:, 3] = ms
plt.scatter(x, y, color=rgba_colors)
plt.axes().set_aspect("equal")
plt.xlim(0, 1)
plt.ylim(0, 1)
writer.add_figure(tag_name, fig, global_step=j)
# Export predictions
if args.export_dir:
os.makedirs(f"{args.export_dir}/groundtruths", exist_ok=True)
os.makedirs(f"{args.export_dir}/detections", exist_ok=True)
for i, (gt, dets) in enumerate(zip(true_export, pred_export)):
with open(f"{args.export_dir}/groundtruths/{i}.txt",
"w") as fd:
for box in gt.transpose(0, 1):
if (box == 0).all():
continue
s = "box " + " ".join(map(str, box.tolist()))
fd.write(s + "\n")
if args.export_progress:
for step, det in enumerate(dets):
with open(
f"{args.export_dir}/detections/{i}-step{step}.txt",
"w") as fd:
for sbox in det.transpose(0, 1):
s = f"box " + " ".join(map(str, sbox.tolist()))
fd.write(s + "\n")
with open(f"{args.export_dir}/detections/{i}.txt", "w") as fd:
for sbox in dets[-1].transpose(0, 1):
s = f"box " + " ".join(map(str, sbox.tolist()))
fd.write(s + "\n")
import subprocess
git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"])
torch.backends.cudnn.benchmark = True
for epoch in range(args.epochs):
tracker.new_epoch()
with mp.Pool(10) as pool:
if not args.eval_only:
run(net,
train_loader,
optimizer,
train=True,
epoch=epoch,
pool=pool)
if not args.train_only:
run(net,
test_loader,
optimizer,
train=False,
epoch=epoch,
pool=pool)
results = {
"name":
args.name,
"tracker":
tracker.data,
"weights":
net.state_dict()
if not args.multi_gpu else net.module.state_dict(),
"args":
vars(args),
"hash":
git_hash,
}
torch.save(results, os.path.join("logs", args.name))
if args.eval_only:
break
def main():
global net
global test_loader
global scatter
parser = argparse.ArgumentParser()
# generic params
parser.add_argument(
"--name",
default=datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
help="Name to store the log file as",
)
parser.add_argument("--resume", help="Path to log file to resume from")
parser.add_argument("--encoder", default="FSEncoder", help="Encoder")
parser.add_argument("--decoder", default="DSPN", help="Decoder")
parser.add_argument("--epochs",
type=int,
default=10,
help="Number of epochs to train with")
parser.add_argument("--latent",
type=int,
default=32,
help="Dimensionality of latent space")
parser.add_argument("--dim",
type=int,
default=64,
help="Dimensionality of hidden layers")
parser.add_argument("--lr",
type=float,
default=1e-2,
help="Outer learning rate of model")
parser.add_argument("--batch-size",
type=int,
default=12,
help="Batch size to train with")
parser.add_argument("--num-workers",
type=int,
default=0,
help="Number of threads for data loader")
parser.add_argument(
"--dataset",
choices=[
"mnist", "clevr-box", "clevr-state", "cats", "merged", "wflw"
],
help="Which dataset to use",
)
parser.add_argument(
"--no-cuda",
action="store_true",
help="Run on CPU instead of GPU (not recommended)",
)
parser.add_argument("--train-only",
action="store_true",
help="Only run training, no evaluation")
parser.add_argument("--eval-only",
action="store_true",
help="Only run evaluation, no training")
parser.add_argument("--multi-gpu",
action="store_true",
help="Use multiple GPUs")
parser.add_argument("--show",
action="store_true",
help="Plot generated samples in Tensorboard")
parser.add_argument(
"--show-skip",
type=int,
default=1,
help="Number of epochs to skip before exporting to Tensorboard")
parser.add_argument(
"--infer-name",
action="store_true",
help="Automatically name run based on dataset/run number")
parser.add_argument("--supervised", action="store_true", help="")
parser.add_argument("--baseline",
action="store_true",
help="Use baseline model")
parser.add_argument("--export-dir",
type=str,
help="Directory to output samples to")
parser.add_argument("--export-n",
type=int,
default=10**9,
help="How many samples to output")
parser.add_argument(
"--export-progress",
action="store_true",
help="Output intermediate set predictions for DSPN?",
)
parser.add_argument(
"--full-eval",
action="store_true",
help="Use full evaluation set (default: 1/10 of evaluation data)",
# don't need full evaluation when training to save some time
)
parser.add_argument(
"--mask-feature",
action="store_true",
help="Treat mask as a feature to compute loss with",
)
parser.add_argument(
"--inner-lr",
type=float,
default=800,
help="Learning rate of DSPN inner optimisation",
)
parser.add_argument(
"--iters",
type=int,
default=10,
help="How many DSPN inner optimisation iteration to take",
)
parser.add_argument(
"--huber-repr",
type=float,
default=1,
help="Scaling of repr loss term for DSPN supervised learning",
)
parser.add_argument(
"--loss",
choices=["hungarian", "chamfer", "emd"],
default="emd",
help="Type of loss used",
)
parser.add_argument(
"--export-csv",
action="store_true",
help="Only perform predictions, don't evaluate in any way")
parser.add_argument("--eval-split", help="Overwrite split on test set")
args = parser.parse_args()
if args.infer_name:
if args.baseline:
prefix = "base"
else:
prefix = "dspn"
used_nums = []
if not os.path.exists("runs"):
os.makedirs("runs")
runs = os.listdir("runs")
for run in runs:
if args.dataset in run:
used_nums.append(int(run.split("-")[-1]))
num = 1
while num in used_nums:
num += 1
name = f"{prefix}-{args.dataset}-{num}"
else:
name = args.name
print(f"Saving run to runs/{name}")
train_writer = SummaryWriter(f"runs/{name}", purge_step=0)
net = model.build_net(args)
if not args.no_cuda:
net = net.cuda()
if args.multi_gpu:
net = torch.nn.DataParallel(net)
optimizer = torch.optim.Adam(
[p for p in net.parameters() if p.requires_grad], lr=args.lr)
print("Building dataloader")
if args.dataset == "mnist":
dataset_train = data.MNISTSet(train=True, full=args.full_eval)
dataset_test = data.MNISTSet(train=False, full=args.full_eval)
elif args.dataset in ["clevr-box", "clevr-state"]:
dataset_train = data.CLEVR("clevr",
"train",
box=args.dataset == "clevr-box",
full=args.full_eval)
dataset_test = data.CLEVR("clevr",
"val",
box=args.dataset == "clevr-box",
full=args.full_eval)
elif args.dataset == "cats":
dataset_train = data.Cats("cats", "train", 9, full=args.full_eval)
dataset_test = data.Cats("cats", "val", 9, full=args.full_eval)
elif args.dataset == "faces":
dataset_train = data.Faces("faces", "train", 4, full=args.full_eval)
dataset_test = data.Faces("faces", "val", 4, full=args.full_eval)
elif args.dataset == "wflw":
if args.eval_split:
eval_split = f"test_{args.eval_split}"
else:
eval_split = "test"
dataset_train = data.WFLW("wflw", "train", 7, full=args.full_eval)
dataset_test = data.WFLW("wflw", eval_split, 7, full=args.full_eval)
elif args.dataset == "merged":
# merged cats and human faces
dataset_train_cats = data.Cats("cats", "train", 9, full=args.full_eval)
dataset_train_wflw = data.WFLW("wflw", "train", 9, full=args.full_eval)
dataset_test_cats = data.Cats("cats", "val", 9, full=args.full_eval)
dataset_test_wflw = data.WFLW("wflw", "test", 9, full=args.full_eval)
dataset_train = data.MergedDataset(dataset_train_cats,
dataset_train_wflw)
dataset_test = data.MergedDataset(dataset_test_cats, dataset_test_wflw)
if not args.eval_only:
train_loader = data.get_loader(dataset_train,
batch_size=args.batch_size,
num_workers=args.num_workers)
if not args.train_only:
test_loader = data.get_loader(dataset_test,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False)
tracker = track.Tracker(
train_mae=track.ExpMean(),
train_last=track.ExpMean(),
train_loss=track.ExpMean(),
test_mae=track.Mean(),
test_last=track.Mean(),
test_loss=track.Mean(),
)
if args.resume:
log = torch.load(args.resume)
weights = log["weights"]
n = net
if args.multi_gpu:
n = n.module
n.load_state_dict(weights, strict=True)
if args.export_csv:
names = []
predictions = []
export_targets = []
def run(net, loader, optimizer, train=False, epoch=0, pool=None):
writer = train_writer
if train:
net.train()
prefix = "train"
torch.set_grad_enabled(True)
else:
net.eval()
prefix = "test"
torch.set_grad_enabled(False)
if args.export_dir:
true_export = []
pred_export = []
iters_per_epoch = len(loader)
loader = tqdm(
loader,
ncols=0,
desc="{1} E{0:02d}".format(epoch, "train" if train else "test "),
)
for i, sample in enumerate(loader, start=epoch * iters_per_epoch):
# input is either a set or an image
input, target_set, target_mask = map(lambda x: x.cuda(), sample)
# forward evaluation through the network
(progress, masks, evals,
gradn), (y_enc, y_label) = net(input, target_set, target_mask)
progress_only = progress
# if using mask as feature, concat mask feature into progress
if args.mask_feature:
target_set = torch.cat(
[target_set, target_mask.unsqueeze(dim=1)], dim=1)
progress = [
torch.cat([p, m.unsqueeze(dim=1)], dim=1)
for p, m in zip(progress, masks)
]
if args.loss == "chamfer":
# dim 0 is over the inner iteration steps
# target set is broadcasted over dim 0
set_loss = utils.chamfer_loss(torch.stack(progress),
target_set.unsqueeze(0))
elif args.loss == "hungarian":
set_loss = utils.hungarian_loss(progress[-1],
target_set,
thread_pool=pool).unsqueeze(0)
elif args.loss == "emd":
set_loss = utils.emd(progress[-1], target_set).unsqueeze(0)
# Only use representation loss with DSPN and when doing general
# supervised prediction, not when auto-encoding
if args.supervised and not args.baseline:
repr_loss = args.huber_repr * F.smooth_l1_loss(y_enc, y_label)
loss = set_loss.mean() + repr_loss.mean()
else:
loss = set_loss.mean()
# restore progress variable to not contain masks for correct
# exporting
progress = progress_only
# Outer optim step
if train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Tensorboard tracking of metrics for debugging
tracked_last = tracker.update(f"{prefix}_last",
set_loss[-1].item())
tracked_loss = tracker.update(f"{prefix}_loss", loss.item())
if train:
writer.add_scalar("metric/set-loss",
loss.item(),
global_step=i)
writer.add_scalar("metric/set-last",
set_loss[-1].mean().item(),
global_step=i)
if not args.baseline:
writer.add_scalar("metric/eval-first",
evals[0].mean().item(),
global_step=i)
writer.add_scalar("metric/eval-last",
evals[-1].mean().item(),
global_step=i)
writer.add_scalar("metric/max-inner-grad-norm",
max(g.item() for g in gradn),
global_step=i)
writer.add_scalar("metric/mean-inner-grad-norm",
sum(g.item()
for g in gradn) / len(gradn),
global_step=i)
if args.supervised:
writer.add_scalar("metric/repr_loss",
repr_loss.item(),
global_step=i)
# Print current progress to progress bar
fmt = "{:.6f}".format
loader.set_postfix(last=fmt(tracked_last),
loss=fmt(tracked_loss),
bad=fmt(evals[-1].detach().cpu().item() *
1000) if not args.baseline else 0)
if args.export_dir:
# export last inner optim of each input as csv
# (one input per row)
if args.export_csv:
# the second to last element are the last of the
# inner optim
for batch_i, p in enumerate(progress[-2]):
img_id = i * args.batch_size + batch_i
names.append(loader.iterable.dataset.get_fname(img_id))
m = masks[-2][batch_i]
m = m.cpu().detach().numpy().astype(bool)
p = p.cpu().detach().numpy()
p = p[:, m]
sample_preds = [
p[k % 2, k // 2] for k in range(p.shape[1] * 2)
]
# remove values according to mask and add zeros to the
# end in stead
sample_preds += [0] * (len(m) * 2 - len(sample_preds))
predictions.append(sample_preds)
true_mask = target_set[batch_i, 2, :].cpu().detach()
true_mask = true_mask.numpy().astype(bool)
trues = target_set[batch_i, :2, :]
trues = trues.cpu().detach().numpy()
t = trues[:, true_mask]
t = [t[k % 2, k // 2] for k in range(t.shape[1] * 2)]
t += [0] * (len(true_mask) * 2 - len(t))
export_targets.append(t)
# Store predictions to be exported
else:
if len(true_export) < args.export_n:
for p, m in zip(target_set, target_mask):
true_export.append(p.detach().cpu())
progress_steps = []
for pro, ms in zip(progress, masks):
# pro and ms are one step of the inner optim
# score boxes contains the list of predicted
# elements for one step
score_boxes = []
for p, m in zip(pro.cpu().detach(),
ms.cpu().detach()):
score_box = torch.cat([m.unsqueeze(0), p],
dim=0)
score_boxes.append(score_box)
progress_steps.append(score_boxes)
for b in zip(*progress_steps):
pred_export.append(b)
# Plot predictions in Tensorboard
if args.show and epoch % args.show_skip == 0 and not train:
name = f"set/epoch-{epoch}/img-{i}"
# thresholded set
progress.append(progress[-1])
masks.append((masks[-1] > 0.5).float())
# target set
if args.mask_feature:
# target set is augmented with masks, so remove them
progress.append(target_set[:, :-1])
else:
progress.append(target_set)
masks.append(target_mask)
# intermediate sets
for j, (s, ms) in enumerate(zip(progress, masks)):
if args.dataset == "clevr-state":
continue
if args.dataset.startswith("clevr"):
threshold = 0.5
else:
threshold = None
s, ms = utils.scatter_masked(
s,
ms,
binned=args.dataset.startswith("clevr"),
threshold=threshold)
if j != len(progress) - 1:
tag_name = f"{name}"
else:
tag_name = f"{name}-target"
if args.dataset == "clevr-box":
img = input[0].detach().cpu()
writer.add_image_with_boxes(tag_name,
img,
s.transpose(0, 1),
global_step=j)
elif args.dataset == "cats" \
or args.dataset == "wflw" \
or args.dataset == "merged":
img = input[0].detach().cpu()
fig = plt.figure()
plt.scatter(s[0, :] * 128, s[1, :] * 128)
plt.imshow(np.transpose(img, (1, 2, 0)))
writer.add_figure(tag_name, fig, global_step=j)
else: # mnist
fig = plt.figure()
y, x = s
y = 1 - y
ms = ms.numpy()
rgba_colors = np.zeros((ms.size, 4))
rgba_colors[:, 2] = 1.0
rgba_colors[:, 3] = ms
plt.scatter(x, y, color=rgba_colors)
plt.axes().set_aspect("equal")
plt.xlim(0, 1)
plt.ylim(0, 1)
writer.add_figure(tag_name, fig, global_step=j)
# Export predictions
if args.export_dir and not args.export_csv:
os.makedirs(f"{args.export_dir}/groundtruths", exist_ok=True)
os.makedirs(f"{args.export_dir}/detections", exist_ok=True)
for i, (gt, dets) in enumerate(zip(true_export, pred_export)):
export_groundtruths_path = os.path.join(
args.export_dir, "groundtruths", f"{i}.txt")
with open(export_groundtruths_path, "w") as fd:
for box in gt.transpose(0, 1):
if (box == 0).all():
continue
s = "box " + " ".join(map(str, box.tolist()))
fd.write(s + "\n")
if args.export_progress:
for step, det in enumerate(dets):
export_progress_path = os.path.join(
args.export_dir, "detections",
f"{i}-step{step}.txt")
with open(export_progress_path, "w") as fd:
for sbox in det.transpose(0, 1):
s = f"box " + " ".join(map(str, sbox.tolist()))
fd.write(s + "\n")
export_path = os.path.join(args.export_dir, "detections",
f"{i}.txt")
with open(export_path, "w") as fd:
for sbox in dets[-1].transpose(0, 1):
s = f"box " + " ".join(map(str, sbox.tolist()))
fd.write(s + "\n")
import subprocess
git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"])
# git_hash = "483igtrfiuey46"
torch.backends.cudnn.benchmark = True
metrics = {}
start = time.time()
if args.eval_only:
tracker.new_epoch()
with mp.Pool(10) as pool:
run(net, test_loader, optimizer, train=False, epoch=0, pool=pool)
metrics["test_loss"] = np.mean(tracker.data["test_loss"][-1])
metrics["test_set_loss"] = np.mean(tracker.data["test_last"][-1])
else:
best_test_loss = float("inf")
for epoch in range(args.epochs):
tracker.new_epoch()
with mp.Pool(10) as pool:
run(net,
train_loader,
optimizer,
train=True,
epoch=epoch,
pool=pool)
if not args.train_only:
run(net,
test_loader,
optimizer,
train=False,
epoch=epoch,
pool=pool)
epoch_test_loss = np.mean(tracker.data["test_loss"][-1])
if epoch_test_loss < best_test_loss:
print("new best loss")
best_test_loss = epoch_test_loss
# only save if the epoch has lower loss
metrics["test_loss"] = epoch_test_loss
metrics["train_loss"] = np.mean(tracker.data["train_loss"][-1])
metrics["train_set_loss"] = np.mean(
tracker.data["train_last"][-1])
metrics["test_set_loss"] = np.mean(
tracker.data["test_last"][-1])
metrics["best_epoch"] = epoch
results = {
"name":
name + "-best",
"tracker":
tracker.data,
"weights":
net.state_dict()
if not args.multi_gpu else net.module.state_dict(),
"args":
vars(args),
"hash":
git_hash,
}
torch.save(results, os.path.join("logs", name + "-best"))
results = {
"name":
name + "-final",
"tracker":
tracker.data,
"weights":
net.state_dict()
if not args.multi_gpu else net.module.state_dict(),
"args":
vars(args),
"hash":
git_hash,
}
torch.save(results, os.path.join("logs", name + "-final"))
if args.export_csv and args.export_dir:
path = os.path.join(args.export_dir, f'{args.name}-predictions.csv')
pd.DataFrame(np.array(predictions), index=names).to_csv(path,
sep=',',
index=names,
header=False)
path = os.path.join(args.export_dir, f'{args.name}-targets.csv')
pd.DataFrame(np.array(export_targets),
index=names).to_csv(path,
sep=',',
index=names,
header=False)
took = time.time() - start
print(f"Process took {took:.1f}s, avg {took/args.epochs:.1f} s/epoch.")
# save hyper parameters to tensorboard for reference
hparams = {k: v for k, v in vars(args).items() if v is not None}
print(metrics)
metrics = {"total_time": took, "avg_time_per_epoch": took / args.epochs}
print("writing hparams")
train_writer.add_hparams(hparams, metric_dict=metrics, name="hparams")
cv2.IMREAD_COLOR)
if next_batch.offset % 2 == 0:
X_batch.append(
model.preprocess(cv2.flip(im, 1), config.INPUT_IMAGE_CROP))
Y_batch.append(-Y[next_batch.offset])
else:
X_batch.append(model.preprocess(im, config.INPUT_IMAGE_CROP))
Y_batch.append(Y[next_batch.offset])
next_batch.offset += 1
return X_batch, Y_batch
X, keep_prob, pred = model.build_net()
Y = tf.placeholder(tf.float32, [None], "human_data")
loss = model.build_loss(Y, pred, config.REGULARIZER_COEF,
tf.trainable_variables())
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name="global_step")
optimizer = tf.train.AdamOptimizer(1e-6, name="optimizer").minimize(
loss, global_step=global_step)
samples, labels = load_training_data()
tf.summary.scalar("loss", loss)
summary_op = tf.summary.merge_all()
train=False,
full=args.full_eval,
max_size=max_set_size)
dataset_val = data.TestDataset(pairPath,
valNum,
args.batch_size,
valLineNums,
vNum,
train=False,
full=args.full_eval,
max_size=max_set_size)
print("max_set_size {}".format(max_set_size))
args.set_size = max_set_size
args.vNum = vNum
net = model.build_net(args)
if not args.no_cuda:
net = net.cuda()
if args.multi_gpu:
net = torch.nn.DataParallel(net)
optimizer = torch.optim.Adam([p for p in net.parameters() if p.requires_grad],
lr=args.lr,
weight_decay=0.01)
decayRate = 0.9
my_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer,
gamma=decayRate)
#optimizer = torch.optim.SGD([p for p in net.parameters() if p.requires_grad], lr=args.lr)
print("valid: ", len(x_valid))
learning_rate = 0.0001
epochs = 2
batch_size = 10
# dataset 생성
train_dataset = data.Dataset_image([x_train, y_train],
batch_size=batch_size)
valid_dataset = data.Dataset_image([x_valid, y_valid],
batch_size=batch_size)
sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True)))
model = model.Vgg(sess, "model")
model.build_net(image_shape=[150, 150, 3], class_count=2)
# 학습 및 평가 수행
train(model, train_dataset, valid_dataset, learning_rate, epochs)
eval(model, valid_dataset)
"""
predict 예제
"""
import pandas as pd
import re
TEST_DIR = DATA_DIR + "test_resize/"
test_images = [TEST_DIR + i for i in os.listdir(TEST_DIR)]
test_images.sort(key=lambda var: [
int(x) if x.isdigit() else x
for x in re.findall(r'[^0-9]|[0-9]+', var)
type=bool,
default=False,
help='Use tensorborad to show the Loss Graph')
args = parser.parse_args()
print_info(
'----------------------------------------------------------------------\n'
'| Training. |\n'
'----------------------------------------------------------------------',
['yellow', 'bold'])
logger = set_logger(args.tensorboard)
global cfg
cfg = Config.fromfile(args.config)
net = build_net(
'train',
size=cfg.model.input_size, # Only 320, 512, 704 and 800 are supported
config=cfg.model.m2det_config)
init_net(net, cfg, args.resume_net
) # init the network with pretrained weights or resumed weights
if args.ngpu > 1:
net = torch.nn.DataParallel(net)
if cfg.train_cfg.cuda:
net.cuda()
cudnn.benchmark = True
optimizer = set_optimizer(net, cfg)
criterion = set_criterion(cfg)
priorbox = PriorBox(anchors(cfg))
with torch.no_grad():
def main(_):
''' Train model from here'''
if not os.path.exists(FLAGS.train_dir):
os.mkdir(FLAGS.train_dir)
print('Created training directory', FLAGS.train_dir)
# get data from data_reader
actual_in_dim = eval(FLAGS.input_dim)
actual_out_dim = eval(FLAGS.output_dim)
data_tensors = load_data(FLAGS.data_dir)
train_reader = DataReader(data_tensors['train'], actual_in_dim,
actual_out_dim, FLAGS.batch_size)
test_reader = DataReader(data_tensors['test'], actual_in_dim,
actual_out_dim, FLAGS.batch_size)
print('initialized all dataset readers')
# build the model
xs = tf.placeholder(tf.float32, [None, len(actual_in_dim)])
ys = tf.placeholder(tf.float32, [None, len(actual_out_dim)])
prediction = build_net(xs, len(actual_in_dim), len(actual_out_dim),
eval(FLAGS.layers), eval(FLAGS.act_funcs))
loss = tf.reduce_mean(
tf.reduce_sum(tf.square(ys - prediction), reduction_indices=[1]))
learning_rate = tf.Variable(FLAGS.learning_rate)
train_step = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(loss)
# initialization and train
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
epoch_start_time = time.time()
# training starts here
for epoch in range(FLAGS.max_epochs):
for x, y in train_reader.iter():
sess.run(train_step, feed_dict={xs: x, ys: y})
cur_loss = float(
sess.run(loss,
feed_dict={
xs: data_tensors['train'][:, actual_in_dim],
ys: data_tensors['train'][:, actual_out_dim]
}))
current_learning_rate = sess.run(learning_rate)
if abs(cur_loss) < current_learning_rate * FLAGS.decay_when:
print("current learningRate: ", current_learning_rate)
change_learn = tf.assign(
learning_rate,
current_learning_rate * FLAGS.learning_rate_decay)
sess.run(change_learn)
print("new learning rate is: ", sess.run(learning_rate))
if epoch % FLAGS.print_every == 0:
time_elapsed = time.time() - epoch_start_time
epoch_start_time = time.time()
print('epoch %7d: train_loss = %6.8f, secs = %.4fs' %
(epoch, cur_loss, time_elapsed))
# test starts here
sess.run(train_step,
feed_dict={
xs: data_tensors['test'][:, actual_in_dim],
ys: data_tensors['test'][:, actual_out_dim]
})
cur_loss = float(
sess.run(loss,
feed_dict={
xs: data_tensors['test'][:, actual_in_dim],
ys: data_tensors['test'][:, actual_out_dim]
}))
prediction_value = sess.run(prediction,
feed_dict={
xs:
data_tensors['test'][:, actual_in_dim],
ys:
data_tensors['test'][:, actual_out_dim]
})
print('testing loss:', cur_loss)
np.savetxt(
"cv/pred_" + output_names[actual_out_dim[0] - len(input_names)] +
"_np_test_loss" + str(cur_loss) + "_" + str(FLAGS.batch_size) +
".txt", prediction_value)
def main():
global net
global test_loader
global scatter
parser = argparse.ArgumentParser()
# generic params
parser.add_argument(
"--name",
default=datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
help="Name to store the log file as",
)
parser.add_argument("--resume", help="Path to log file to resume from")
parser.add_argument("--encoder", default="FSEncoder", help="Encoder")
parser.add_argument("--decoder", default="DSPN", help="Decoder")
parser.add_argument(
"--epochs", type=int, default=10, help="Number of epochs to train with"
)
parser.add_argument(
"--latent", type=int, default=32, help="Dimensionality of latent space"
)
parser.add_argument(
"--dim", type=int, default=64, help="Dimensionality of hidden layers"
)
parser.add_argument(
"--lr", type=float, default=1e-2, help="Outer learning rate of model"
)
parser.add_argument(
"--batch-size", type=int, default=32, help="Batch size to train with"
)
parser.add_argument(
"--num-workers", type=int, default=2, help="Number of threads for data loader"
)
parser.add_argument(
"--dataset",
choices=["mnist", "clevr-box", "clevr-state", "lhc"],
help="Use MNIST dataset",
)
parser.add_argument(
"--no-cuda",
action="store_true",
help="Run on CPU instead of GPU (not recommended)",
)
parser.add_argument(
"--train-only", action="store_true", help="Only run training, no evaluation"
)
parser.add_argument(
"--eval-only", action="store_true", help="Only run evaluation, no training"
)
parser.add_argument("--multi-gpu", action="store_true", help="Use multiple GPUs")
parser.add_argument(
"--show", action="store_true", help="Plot generated samples in Tensorboard"
)
parser.add_argument("--supervised", action="store_true", help="")
parser.add_argument("--baseline", action="store_true", help="Use baseline model")
parser.add_argument("--export-dir", type=str, help="Directory to output samples to")
parser.add_argument(
"--export-n", type=int, default=10 ** 9, help="How many samples to output"
)
parser.add_argument(
"--export-progress",
action="store_true",
help="Output intermediate set predictions for DSPN?",
)
parser.add_argument(
"--full-eval",
action="store_true",
help="Use full evaluation set (default: 1/10 of evaluation data)", # don't need full evaluation when training to save some time
)
parser.add_argument(
"--mask-feature",
action="store_true",
help="Treat mask as a feature to compute loss with",
)
parser.add_argument(
"--inner-lr",
type=float,
default=800,
help="Learning rate of DSPN inner optimisation",
)
parser.add_argument(
"--iters",
type=int,
default=10,
help="How many DSPN inner optimisation iteration to take",
)
parser.add_argument(
"--huber-repr",
type=float,
default=1,
help="Scaling of representation loss term for DSPN supervised learning",
)
parser.add_argument(
"--loss",
choices=["hungarian", "chamfer"],
default="hungarian",
help="Type of loss used",
)
args = parser.parse_args()
train_writer = SummaryWriter(f"runs/{args.name}", purge_step=0)
net = model.build_net(args)
if not args.no_cuda:
net = net.cuda()
if args.multi_gpu:
net = torch.nn.DataParallel(net)
optimizer = torch.optim.Adam(
[p for p in net.parameters() if p.requires_grad], lr=args.lr
)
if args.dataset == "mnist":
dataset_train = data.MNISTSet(train=True, full=args.full_eval)
dataset_test = data.MNISTSet(train=False, full=args.full_eval)
elif args.dataset == "lhc":
dataset_train = data.LHCSet(train=True)
dataset_test = data.LHCSet(train=False)
else:
dataset_train = data.CLEVR(
"clevr", "train", box=args.dataset == "clevr-box", full=args.full_eval
)
dataset_test = data.CLEVR(
"clevr", "val", box=args.dataset == "clevr-box", full=args.full_eval
)
if not args.eval_only:
train_loader = data.get_loader(
dataset_train, batch_size=args.batch_size, num_workers=args.num_workers
)
if not args.train_only:
test_loader = data.get_loader(
dataset_test,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False,
)
tracker = track.Tracker(
train_mae=track.ExpMean(),
train_last=track.ExpMean(),
train_loss=track.ExpMean(),
test_mae=track.Mean(),
test_last=track.Mean(),
test_loss=track.Mean(),
)
if args.resume:
log = torch.load(args.resume)
weights = log["weights"]
n = net
if args.multi_gpu:
n = n.module
n.load_state_dict(weights, strict=True)
def run(net, loader, optimizer, train=False, epoch=0, pool=None):
writer = train_writer
if train:
net.train()
prefix = "train"
torch.set_grad_enabled(True)
else:
net.eval()
prefix = "test"
torch.set_grad_enabled(False)
total_train_steps = args.epochs * len(loader)
if args.export_dir:
true_export = []
pred_export = []
iters_per_epoch = len(loader) #len(loader) is # batches, which is len(dataset)/batch size (which is an arg)
loader = tqdm(
loader,
ncols=0,
desc="{1} E{0:02d}".format(epoch, "train" if train else "test "),
)
losses = []
steps = []
encodings = []
for i, sample in enumerate(loader, start=epoch * iters_per_epoch):
# input is either a set or an image
input, target_set, target_mask = map(lambda x: x.cuda(), sample)
# forward evaluation through the network
(progress, masks, evals, gradn), (y_enc, y_label) = net(
input, target_set, target_mask
)
encodings.append(y_label)
progress_only = progress
# if using mask as feature, concat mask feature into progress
if args.mask_feature:
target_set = torch.cat(
[target_set, target_mask.unsqueeze(dim=1)], dim=1
)
progress = [
torch.cat([p, m.unsqueeze(dim=1)], dim=1)
for p, m in zip(progress, masks)
]
if args.loss == "chamfer":
# dim 0 is over the inner iteration steps
# target set is broadcasted over dim 0
set_loss = utils.chamfer_loss(
torch.stack(progress), target_set.unsqueeze(0)
)
else:
# dim 0 is over the inner iteration steps
a = torch.stack(progress)
# target set is explicitly broadcasted over dim 0
b = target_set.repeat(a.size(0), 1, 1, 1)
# flatten inner iteration dim and batch dim
a = a.view(-1, a.size(2), a.size(3))
b = b.view(-1, b.size(2), b.size(3))
set_loss = utils.hungarian_loss(
progress[-1], target_set, thread_pool=pool
).unsqueeze(0)
# Only use representation loss with DSPN and when doing general supervised prediction, not when auto-encoding
if args.supervised and not args.baseline:
repr_loss = args.huber_repr * F.smooth_l1_loss(y_enc, y_label)
loss = set_loss.mean() + repr_loss.mean()
else:
loss = set_loss.mean()
# restore progress variable to not contain masks for correct exporting
progress = progress_only
# Outer optim step
if train:
optimizer.zero_grad()
loss.backward()
optimizer.step()
# loss value can be gotten as loss.item() for graphing purposes
steps.append(i)
losses.append(loss.item())
# Tensorboard tracking of metrics for debugging
tracked_last = tracker.update("{}_last".format(prefix), set_loss[-1].item())
tracked_loss = tracker.update("{}_loss".format(prefix), loss.item())
if train:
writer.add_scalar("metric/set-loss", loss.item(), global_step=i)
writer.add_scalar(
"metric/set-last", set_loss[-1].mean().item(), global_step=i
)
if not args.baseline:
writer.add_scalar(
"metric/eval-first", evals[0].mean().item(), global_step=i
)
writer.add_scalar(
"metric/eval-last", evals[-1].mean().item(), global_step=i
)
writer.add_scalar(
"metric/max-inner-grad-norm",
max(g.item() for g in gradn),
global_step=i,
)
writer.add_scalar(
"metric/mean-inner-grad-norm",
sum(g.item() for g in gradn) / len(gradn),
global_step=i,
)
if args.supervised:
writer.add_scalar(
"metric/repr_loss", repr_loss.item(), global_step=i
)
# Print current progress to progress bar
fmt = "{:.6f}".format
loader.set_postfix(
last=fmt(tracked_last),
loss=fmt(tracked_loss),
bad=fmt(evals[-1].detach().cpu().item() * 1000)
if not args.baseline
else 0,
)
# Store predictions to be exported
if args.export_dir:
if len(true_export) < args.export_n:
for p, m in zip(target_set, target_mask):
true_export.append(p.detach().cpu())
progress_steps = []
for pro, mas in zip(progress, masks):
# pro and mas are one step of the inner optim
# score boxes contains the list of predicted elements for one step
score_boxes = []
for p, m in zip(pro.cpu().detach(), mas.cpu().detach()):
score_box = torch.cat([m.unsqueeze(0), p], dim=0)
score_boxes.append(score_box)
progress_steps.append(score_boxes)
for b in zip(*progress_steps):
pred_export.append(b)
# Plot predictions in Tensorboard
if args.show and not train:
name = f"set/epoch-{epoch}/img-{i}"
# thresholded set
progress.append(progress[-1])
masks.append((masks[-1] > 0.5).float())
# target set
if args.mask_feature:
# target set is augmented with masks, so remove them
progress.append(target_set[:, :-1])
else:
progress.append(target_set)
masks.append(target_mask)
# intermediate sets
for j, (s, ms) in enumerate(zip(progress, masks)):
if args.dataset == "clevr-state":
continue
s, ms = utils.scatter_masked(
s,
ms,
binned=args.dataset.startswith("clevr"),
threshold=0.5 if args.dataset.startswith("clevr") else None,
)
tag_name = f"{name}" if j != len(progress) - 1 else f"{name}-target"
if args.dataset == "clevr-box":
img = input[0].detach().cpu()
writer.add_image_with_boxes(
tag_name, img, s.transpose(0, 1), global_step=j
)
elif args.dataset == "clevr-state":
pass
else: # mnist
fig = plt.figure()
y, x = s
y = 1 - y
ms = ms.numpy()
rgba_colors = np.zeros((ms.size, 4))
rgba_colors[:, 2] = 1.0
rgba_colors[:, 3] = ms
plt.scatter(x, y, color=rgba_colors)
plt.axes().set_aspect("equal")
plt.xlim(0, 1)
plt.ylim(0, 1)
writer.add_figure(tag_name, fig, global_step=j)
#create scatter of encoded points?
print([t.shape for t in encodings[0:5]])
'''
#get the first latent dimension, and plot that
first_latent_dim = [t.detach().cpu()[0] for t in encodings[100:]]
numbers = list(range(len(first_latent_dim[0]))) * len(first_latent_dim)
first_latent_dim_as_list = []
for tensor in first_latent_dim:
for entry in tensor:
first_latent_dim_as_list.append(entry)
print(len(numbers))
print(len(first_latent_dim_as_list))
fig = plt.figure()
#plt.yscale('log')
#print(first_latent_dim[0:5])
plt.scatter(numbers, first_latent_dim_as_list)
name = f'img-latent-epoch-{epoch}-{"train" if train else "test"}'
plt.savefig(name, dpi=300)
plt.close(fig)
'''
#create scatter of encoded points, put through tsne
# see: https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
set_encoding_in_hidden_dim = torch.cat(encodings, dim = 0).detach().cpu()
set_encoding_tsne = TSNE(n_components = 2).fit_transform(set_encoding_in_hidden_dim)
set_encoding_tsne_x = set_encoding_tsne[:, 0]
set_encoding_tsne_y = set_encoding_tsne[:, 1]
fig = plt.figure()
plt.scatter(set_encoding_tsne_x, set_encoding_tsne_y)
name = f'img-latent-epoch-{epoch}-{"train" if train else "test"}'
plt.savefig(name, dpi=300)
plt.close(fig)
# create graph
fig = plt.figure()
#plt.yscale('linear')
plt.scatter(steps, losses)
name = f"img-losses-epoch-{epoch}-train-{'train' if train else 'test'}"
plt.savefig(name, dpi=300)
plt.close(fig)
# Export predictions
if args.export_dir:
os.makedirs(f"{args.export_dir}/groundtruths", exist_ok=True)
os.makedirs(f"{args.export_dir}/detections", exist_ok=True)
for i, (gt, dets) in enumerate(zip(true_export, pred_export)):
with open(f"{args.export_dir}/groundtruths/{i}.txt", "w") as fd:
for box in gt.transpose(0, 1):
if (box == 0).all():
continue
s = "box " + " ".join(map(str, box.tolist()))
fd.write(s + "\n")
if args.export_progress:
for step, det in enumerate(dets):
with open(
f"{args.export_dir}/detections/{i}-step{step}.txt", "w"
) as fd:
for sbox in det.transpose(0, 1):
s = f"box " + " ".join(map(str, sbox.tolist()))
fd.write(s + "\n")
with open(f"{args.export_dir}/detections/{i}.txt", "w") as fd:
for sbox in dets[-1].transpose(0, 1):
s = f"box " + " ".join(map(str, sbox.tolist()))
fd.write(s + "\n")
return np.mean(losses)
import subprocess
git_hash = subprocess.check_output(["git", "rev-parse", "HEAD"])
torch.backends.cudnn.benchmark = True
epoch_losses = []
for epoch in range(args.epochs):
tracker.new_epoch()
with mp.Pool(10) as pool:
if not args.eval_only:
e_loss = run(net, train_loader, optimizer, train=True, epoch=epoch, pool=pool)
epoch_losses.append(e_loss)
if not args.train_only:
e_loss = run(net, test_loader, optimizer, train=False, epoch=epoch, pool=pool)
results = {
"name": args.name,
"tracker": tracker.data,
"weights": net.state_dict()
if not args.multi_gpu
else net.module.state_dict(),
"args": vars(args),
"hash": git_hash,
}
torch.save(results, os.path.join("logs", args.name))
if args.eval_only:
break
#plot encoding/decoding
#train loader is bkdg points, test loader is signal points
with mp.Pool(10) as pool:
train_encodings = []
for i, sample in enumerate(train_loader):
# input is either a set or an image
input, target_set, target_mask = map(lambda x: x.cuda(), sample)
# forward evaluation through the network
(progress, masks, evals, gradn), (y_enc, y_label) = net(
input, target_set, target_mask
)
train_encodings.append(y_label)
test_encodings = []
for i, sample in enumerate(test_loader):
# input is either a set or an image
input, target_set, target_mask = map(lambda x: x.cuda(), sample)
# forward evaluation through the network
(progress, masks, evals, gradn), (y_enc, y_label) = net(
input, target_set, target_mask
)
test_encodings.append(y_label)
#recall: "train" is actually just bkgd and "test" is really just signal,
#because we're trying to learn the background distribution and then encode signal points in the same way, and see what happens
#so train an MLP and see if it can distinguish
num_bkdg_val = len(train_encodings) // 3
num_sign_val = len(test_encodings) // 3
bkgd_train_encodings_tensor = torch.cat(train_encodings[:-num_bkdg_val], dim = 0)
sign_train_encodings_tensor = torch.cat(test_encodings[:-num_sign_val], dim = 0)
bkgd_train_encoding_label_tensor = torch.zeros((len(train_encodings) - num_bkdg_val) * args.batch_size, dtype=torch.long)
sign_train_encoding_label_tensor = torch.ones((len(test_encodings) - num_sign_val) * args.batch_size, dtype=torch.long)
train_encodings_tensor = torch.cat((bkgd_train_encodings_tensor, sign_train_encodings_tensor)).to('cuda:0')
train_encodings_tensor_copy = torch.tensor(train_encodings_tensor, requires_grad=True).to('cuda:0')
train_labels_tensor = torch.cat((bkgd_train_encoding_label_tensor, sign_train_encoding_label_tensor)).to('cuda:0')
bkgd_test_encodings_tensor = torch.cat(train_encodings[-num_bkdg_val:])
sign_test_encodings_tensor = torch.cat(test_encodings[-num_sign_val:])
bkgd_test_encoding_label_tensor = torch.zeros((num_bkdg_val) * args.batch_size, dtype=torch.long)
sign_test_encoding_label_tensor = torch.ones((num_sign_val) * args.batch_size, dtype=torch.long)
test_encodings_tensor = torch.cat((bkgd_test_encodings_tensor, sign_test_encodings_tensor)).to('cuda:0')
test_encodings_tensor_copy = torch.tensor(test_encodings_tensor, requires_grad=True).to('cuda:0')
test_labels_tensor = torch.cat((bkgd_test_encoding_label_tensor, sign_test_encoding_label_tensor)).to('cuda:0')
_, encoding_len = y_label.shape
mlp = dummymlp.MLP(embedding_dim=encoding_len, hidden_dim=20, label_dim=2).to('cuda:0')
loss_func = nn.CrossEntropyLoss().to('cuda:0')
#mlp.train()
epochs = 20
for epoch in range(epochs):
mlp.zero_grad()
output = mlp(train_encodings_tensor_copy)
loss = loss_func(output, train_labels_tensor)
loss.backward()
#test
#mlp.eval()
test_output = mlp(test_encodings_tensor_copy)
argmaxed_test_output = torch.argmax(test_output, dim=1)
total = argmaxed_test_output.size(0)
correct = (argmaxed_test_output == test_labels_tensor).sum().item()
print(f'acc = {correct / total}')
#create scatter of encoded points, put through tsne
# see: https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
tsne = TSNE(n_components = 2)
num_test = len(test_encodings) * args.batch_size
fig = plt.figure()
set_encoding_in_hidden_dim_train = torch.cat(train_encodings, dim = 0).detach().cpu()
set_encoding_in_hidden_dim_test = torch.cat(test_encodings, dim = 0).detach().cpu()
set_encoding_in_hidden_dim = torch.cat((set_encoding_in_hidden_dim_test, set_encoding_in_hidden_dim_train), dim = 0)
set_encoding_tsne = tsne.fit_transform(set_encoding_in_hidden_dim)
set_encoding_tsne_x_train = set_encoding_tsne[num_test:, 0]
set_encoding_tsne_y_train = set_encoding_tsne[num_test:, 1]
set_encoding_tsne_x_test = set_encoding_tsne[:num_test, 0]
set_encoding_tsne_y_test = set_encoding_tsne[:num_test, 1]
plt.scatter(set_encoding_tsne_x_train, set_encoding_tsne_y_train)
plt.scatter(set_encoding_tsne_x_test, set_encoding_tsne_y_test)
name = f'img-latent-NEW'
plt.savefig(name, dpi=300)
plt.close(fig)
fig = plt.figure()
plt.scatter(range(args.epochs), epoch_losses)
name = f"img-losses-per-epoch-train-{'train' if not args.eval_only else 'test'}"
plt.savefig(name, dpi=300)
