def train_D(data, labels=None, gen_data=None):
if args.debug: print("dtrain")
D.train()
D_optimizer.zero_grad()
run_batch_size = data.shape[0]
deb = run_batch_size != args.batch_size
if gen_data is None:
gen_data = utils.gen(args, G, normal_dist, run_batch_size, labels=labels)
if args.augment:
p = args.aug_prob if not args.adaptive_prob else losses['p'][-1]
data = augment.augment(args, data, p)
gen_data = augment.augment(args, gen_data, p)
D_real_output = D(data.clone(), labels, deb)
if args.debug or run_batch_size != args.batch_size:
print("D real output: ")
print(D_real_output[:10])
D_fake_output = D(gen_data, labels, deb)
if args.debug or run_batch_size != args.batch_size:
print("D fake output: ")
print(D_fake_output[:10])
D_loss, D_loss_items = utils.calc_D_loss(args, D, data, gen_data, D_real_output, D_fake_output, run_batch_size, Y_real, Y_fake)
D_loss.backward()
D_optimizer.step()
return D_loss_items
def train_D(data, gen_data=None, unrolled=False):
if args.debug: print("dtrain")
D.train()
D_optimizer.zero_grad()
run_batch_size = data.shape[0] if not args.gcnn else data.y.shape[0]
if gen_data is None:
gen_data = utils.gen(args, G, normal_dist, run_batch_size)
if (args.gcnn):
gen_data = utils.convert_to_batch(args, gen_data,
run_batch_size)
if args.augment:
p = args.aug_prob if not args.adaptive_prob else losses['p'][-1]
data = augment.augment(args, data, p)
gen_data = augment.augment(args, gen_data, p)
D_real_output = D(data.clone())
D_fake_output = D(gen_data)
D_loss, D_loss_items = utils.calc_D_loss(args, D, data, gen_data,
D_real_output, D_fake_output,
run_batch_size, Y_real,
Y_fake)
D_loss.backward(create_graph=unrolled)
D_optimizer.step()
return D_loss_items
def img_aug(name, train=False, input_shape=(480, 480)):
global number
img = cv2.imread(image_path + name)
if train == False:
h, w = input_shape
ih, iw = img.shape[:2]
scale = min(w / iw, h / ih)
nw = int(iw * scale)
nh = int(ih * scale)
img = cv2.resize(img, (nw, nh))
img = augment(img)
# 用灰色像素块来做背景扩充图片满足输入尺寸需求
dx = (w - nw) // 2
dy = (h - nh) // 2
img = np.pad(img, ((dy, dy), (dx, dx), (0, 0)),
'constant',
constant_values=128)
if tuple(img.shape[:2]) != input_shape:
img = np.pad(img, ((0, input_shape[0] - img.shape[0]),
(0, input_shape[1] - img.shape[1]), (0, 0)),
'constant',
constant_values=128)
else:
img = augment(img)
cv2.imwrite(result_path + name, img)
number += 1
print('{}. {} is finish!'.format(number, name))
def get_weather(year):
weather_file = '/home/malcolm/uclan/output/wparms/weather_parms{}.csv'.format(year)
weather = pd.read_csv(weather_file, header=0, parse_dates=[0], index_col=0)
weather.index = pd.DatetimeIndex(weather.index).tz_localize('UTC')
augment.augment(weather)
daily_weather = weather.resample('D').mean()
return daily_weather
def train(args, model, sess, dataset):
print('-- Train')
for key in ['model', 'log']:
if not os.path.isdir(args.path[key]):
os.makedirs(args.path[key])
saver = tf.train.Saver()
random_state = np.random.RandomState(seed=0)
logs = {'train': {'itr': [], 'los': [], 'acc': []}, 'val': {'itr': [], 'los': [], 'acc': []}}
t_start = time.time()
generators = {}
generators['train'] = dataset.get_generator('unlimited', 'train', True)
generators['val'] = dataset.get_generator('unlimited', 'val', True)
for itr in range(args.train_iterations):
batch = dataset.get_next_batch(args.batch_size, generators['train'])
batch = augment(batch, args.aug_kinds, random_state)
feed_dict = {}
feed_dict.update({model.inputs[key]: batch[key] for key in ['image', 'label']})
feed_dict.update({model.is_train: True})
input_tensors = [model.outputs]
if (itr+1) % args.check_interval == 0:
input_tensors.extend([model.sparsity])
input_tensors.extend([model.train_op])
result = sess.run(input_tensors, feed_dict)
logs['train'] = _update_logs(logs['train'],
{'itr': itr+1, 'los': result[0]['los'], 'acc': result[0]['acc']})
# Check on validation set.
if (itr+1) % args.check_interval == 0:
batch = dataset.get_next_batch(args.batch_size, generators['val'])
batch = augment(batch, args.aug_kinds, random_state)
feed_dict = {}
feed_dict.update({model.inputs[key]: batch[key] for key in ['image', 'label']})
input_tensors = [model.outputs]
result_val = sess.run(input_tensors, feed_dict)
logs['val'] = _update_logs(logs['val'],
{'itr': itr+1, 'los': result_val[0]['los'], 'acc': result_val[0]['acc']})
# Print results
if (itr+1) % args.check_interval == 0:
pstr = '(train/val) los:{:.3f}/{:.3f} acc:{:.3f}/{:.3f} spa:{:.3f}'.format(
result[0]['los'], result_val[0]['los'],
result[0]['acc'], result_val[0]['acc'],
result[1],
)
print('itr{}: {} (t:{:.1f})'.format(itr+1, pstr, time.time() - t_start))
t_start = time.time()
# Save
if (itr+1) % args.save_interval == 0:
saver.save(sess, args.path['model'] + '/itr-' + str(itr))
_save_logs(os.path.join(args.path['log'], 'train.pickle'), logs['train'])
_save_logs(os.path.join(args.path['log'], 'val.pickle'), logs['val'])
def get_prepared_data(image_path_list):
train_X = []
train_y = []
parent_path = Path(image_path_list[0]).parent
scale_folder = os.path.join(parent_path, 'scale')
augmented_folder = os.path.join(parent_path, 'padding', 'output')
for index, image_path in enumerate(image_path_list):
newImg = get_feature_from_link(image_path)
feature = newImg.reshape(2048)
train_X.append(feature)
train_y.append('1')
feature_cut = model.get_feature_vector(sess, cut_block(image_path))
feature_cut = feature_cut.reshape(2048)
train_X.append(feature_cut)
train_y.append('0')
if os.path.isdir(scale_folder):
if Path(image_path).name not in os.listdir(scale_folder):
resize_with_pad(image_path)
scale_image(image_path)
else:
resize_with_pad(image_path)
scale_image(image_path)
if os.path.exists(augmented_folder):
shutil.rmtree(augmented_folder)
augment(Path(augmented_folder).parent)
scaled_images = [
os.path.join(scale_folder, e) for e in os.listdir(scale_folder)
]
augmented_images = [
os.path.join(augmented_folder, e) for e in os.listdir(augmented_folder)
]
# for index, image_path in enumerate(scaled_images):
# newImg = get_feature_from_link(image_path)
# feature = newImg.reshape(2048)
# train_X.append(feature)
# train_y.append('0')
for index, image_path in enumerate(augmented_images):
newImg = get_feature_from_link(image_path)
feature = newImg.reshape(2048)
train_X.append(feature)
train_y.append('0')
train_X = np.asarray(train_X)
train_y = np.asarray(train_y)
return train_X, train_y
def train():
x = tf.placeholder(tf.float32, [None, 96, 96, 3])
t = tf.placeholder(tf.int32, [None])
is_training = tf.placeholder(tf.bool, [])
model = VGG19(x, t, is_training)
sess = tf.Session()
with tf.variable_scope('vgg19'):
global_step = tf.Variable(0, name='global_step', trainable=False)
opt = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = opt.minimize(model.loss, global_step=global_step)
init = tf.global_variables_initializer()
sess.run(init)
# Restore the latest model
if tf.train.get_checkpoint_state('backup/'):
saver = tf.train.Saver()
saver.restore(sess, 'backup/latest')
# Load the dataset
x_train, t_train, x_test, t_test = load.load()
# Train
while True:
epoch = int(sess.run(global_step) / np.ceil(len(x_train)/batch_size)) + 1
print('epoch:', epoch)
perm = np.random.permutation(len(x_train))
x_train = x_train[perm]
t_train = t_train[perm]
sum_loss_value = 0
for i in tqdm(range(0, len(x_train), batch_size)):
x_batch = augment.augment(x_train[i:i+batch_size])
t_batch = t_train[i:i+batch_size]
_, loss_value = sess.run(
[train_op, model.loss],
feed_dict={x: x_batch, t: t_batch, is_training: True})
sum_loss_value += loss_value
print('loss:', sum_loss_value)
saver = tf.train.Saver()
saver.save(sess, 'backup/latest', write_meta_graph=False)
prediction = np.array([])
answer = np.array([])
for i in range(0, len(x_test), batch_size):
x_batch = augment.augment(x_test[i:i+batch_size])
t_batch = t_test[i:i+batch_size]
output = model.out.eval(
feed_dict={x: x_batch, is_training: False}, session=sess)
prediction = np.concatenate([prediction, np.argmax(output, 1)])
answer = np.concatenate([answer, t_batch])
correct_prediction = np.equal(prediction, answer)
accuracy = np.mean(correct_prediction)
print('accuracy:', accuracy)
def train_G(data):
if args.debug: print("gtrain")
G.train()
G_optimizer.zero_grad()
gen_data = utils.gen(args, G, normal_dist, args.batch_size)
if (args.gcnn):
gen_data = utils.convert_to_batch(args, gen_data, args.batch_size)
if args.augment:
p = args.aug_prob if not args.adaptive_prob else losses['p'][-1]
gen_data = augment.augment(args, gen_data, p)
if (args.unrolled_steps > 0):
D_backup = deepcopy(D)
for i in range(args.unrolled_steps - 1):
train_D(data, gen_data=gen_data, unrolled=True)
D_fake_output = D(gen_data)
G_loss = utils.calc_G_loss(args, D_fake_output, Y_real)
G_loss.backward()
G_optimizer.step()
if (args.unrolled_steps > 0):
D.load(D_backup)
return G_loss.item()
def load_images(image_paths, deterministic=False):
X = []
labels = []
for image_path in image_paths:
with gzip.open(image_path) as file:
xy_xz_yz = pickle.load(file)
x, y, z = xy_xz_yz
if P.AUGMENT and not deterministic:
x, y, z = augment.augment(xy_xz_yz)
offset = (len(x) - P.INPUT_SIZE) / 2
if offset > 0:
x = x[offset:-offset, offset:-offset]
y = y[offset:-offset, offset:-offset]
z = z[offset:-offset, offset:-offset]
X.append([x])
X.append([y])
X.append([z])
label = [0, 1] if "True" in image_path else [1, 0]
labels.append(label)
labels.append(label)
labels.append(label)
X = np.array(X)
X = normalize(X)
if P.ZERO_CENTER:
X -= P.MEAN_PIXEL
return np.array(X, dtype=np.float32), np.array(labels)
def train_G(data, labels=None, epoch=0):
logging.debug("gtrain")
G.train()
G_optimizer.zero_grad()
run_batch_size = labels.shape[
0] if labels is not None else args.batch_size
gen_data = utils.gen(args, G, run_batch_size, labels=labels)
if args.augment:
p = args.aug_prob if not args.adaptive_prob else losses['p'][-1]
gen_data = augment.augment(args, gen_data, p)
D_fake_output = D(gen_data, labels, epoch=epoch)
logging.debug("D fake output:")
logging.debug(D_fake_output[:10])
G_loss = utils.calc_G_loss(args, D_fake_output, Y_real, run_batch_size,
mse)
G_loss.backward()
G_optimizer.step()
return G_loss.item()
def getRandomPositivePatch(x_full, y_full):
x_i, y_i = getRandomPositiveImage(x_full, y_full)
x_s, y_s = getRandomPositiveSlices(x_i, y_i)
x_aug, y_aug = augment(x_s, y_s)
x_patch, y_patch, found = getRandomPositivePatchAllSlices(x_aug, y_aug)
if not found:
x_patch, y_patch = getRandomPositivePatch(x_full, y_full)
return x_patch, y_patch
def next_batch(self, num, data, label):
idx = np.arange(0, len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = []
label_shuffle = []
for i in idx:
distorted_data = augment(data[i])
data_shuffle.append(distorted_data)
label_shuffle.append(label[i])
return np.asarray(data_shuffle), np.asarray(label_shuffle)
def train_D(data, labels=None, gen_data=None, epoch=0, print_output=False):
logging.debug("dtrain")
log = logging.info if print_output else logging.debug
D.train()
D_optimizer.zero_grad()
G.eval()
run_batch_size = data.shape[0]
D_real_output = D(data.clone(), labels, epoch=epoch)
log("D real output: ")
log(D_real_output[:10])
if gen_data is None:
gen_data = utils.gen(args, G, run_batch_size, labels=labels)
if args.augment:
p = args.aug_prob if not args.adaptive_prob else losses['p'][-1]
data = augment.augment(args, data, p)
gen_data = augment.augment(args, gen_data, p)
log("G output: ")
log(gen_data[:2, :10, :])
D_fake_output = D(gen_data, labels, epoch=epoch)
log("D fake output: ")
log(D_fake_output[:10])
D_loss, D_loss_items = utils.calc_D_loss(args, D, data, gen_data,
D_real_output, D_fake_output,
run_batch_size, Y_real,
Y_fake, mse)
D_loss.backward()
D_optimizer.step()
return D_loss_items
def __getitem__(self, idx):
img, label_idx = self._raw_data[idx]
img = np.reshape(img, (3, 32, 32))
img = img.transpose([1, 2, 0])
img = augment(img)
img = img.transpose([2,0,1])
# TODO: Augmentation
img = img / 255.0
img = (img-self._mean)/self._std
label = np.zeros(10)
label[label_idx] = 1
return img, label
def put(self, q):
queue_max_size = 1000
paths = []
while True:
if len(paths) == 0:
paths = self.get_paths()
if q.qsize() >= queue_max_size:
time.sleep(0.1)
continue
ix = np.random.randint(0, len(paths))
path = paths.pop(ix)
im_path, bb_path = path
npimg = np.fromfile(im_path, dtype=np.uint8)
img = cv2.imdecode(npimg, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
with open(bb_path) as f:
lines = f.read().splitlines()
boxes = []
for line in lines:
ix, xmin, ymin, xmax, ymax = line.split('\t')
onehot_label = np.eye(self.num_classes)[int(ix)]
box = [float(xmin),
float(ymin),
float(xmax),
float(ymax)] + onehot_label.tolist()
boxes.append(box)
img, boxes = augment(img, boxes, self.input_size)
if len(boxes) == 0:
continue
boxes = np.array(boxes)
assignment = assign_boxes(boxes, self.priors, self.num_classes,
self.assignment_threshold)
q.put([img, assignment])
def gen(samples, batch_size=32, aug = False):
while 1: # Loop forever so the generator never terminates
shuffle(samples)
images = []
labels = []
for sample in samples:
# Extract steering
steering = preprocess_steering(float(sample["steering"]))
# Randomly drop near zero steering values
if(abs(steering) < 0.1 and np.random.rand() < 0.7):
continue
# read in image
image = imageio.imread(cache_dir + sample["img_file"])
# Augment data with noise and lighting changes
if aug:
image = augment(image)
# Prepare image for network
image = preprocess_camera(image)
images.append(image)
labels.append(steering)
#return batch
if(len(images) >= batch_size):
X = np.array(images)
y = np.array(labels)
images = []
labels = []
yield shuffle(X,y)
def train(args, loader, generator, discriminator, contrast_learner, g_optim,
d_optim, g_ema):
if args.distributed:
g_module = generator.module
d_module = discriminator.module
if contrast_learner is not None:
cl_module = contrast_learner.module
else:
g_module = generator
d_module = discriminator
cl_module = contrast_learner
loader = sample_data(loader)
sample_z = th.randn(args.n_sample, args.latent_size, device=device)
mse = th.nn.MSELoss()
mean_path_length = 0
ada_augment = th.tensor([0.0, 0.0], device=device)
ada_aug_p = args.augment_p if args.augment_p > 0 else 0.0
ada_aug_step = args.ada_target / args.ada_length
r_t_stat = 0
fids = []
pbar = range(args.iter)
if get_rank() == 0:
pbar = tqdm(pbar,
initial=args.start_iter,
dynamic_ncols=True,
smoothing=0)
for idx in pbar:
i = idx + args.start_iter
if i > args.iter:
print("Done!")
break
loss_dict = {
"Generator": th.tensor(0, device=device).float(),
"Discriminator": th.tensor(0, device=device).float(),
"Real Score": th.tensor(0, device=device).float(),
"Fake Score": th.tensor(0, device=device).float(),
"Contrastive": th.tensor(0, device=device).float(),
"Consistency": th.tensor(0, device=device).float(),
"R1 Penalty": th.tensor(0, device=device).float(),
"Path Length Regularization": th.tensor(0, device=device).float(),
"Augment": th.tensor(0, device=device).float(),
"Rt": th.tensor(0, device=device).float(),
}
requires_grad(generator, False)
requires_grad(discriminator, True)
discriminator.zero_grad()
for _ in range(args.num_accumulate):
real_img_og = next(loader).to(device)
noise = make_noise(args.batch_size, args.latent_size,
args.mixing_prob)
fake_img_og, _ = generator(noise)
if args.augment:
fake_img, _ = augment(fake_img_og, ada_aug_p)
real_img, _ = augment(real_img_og, ada_aug_p)
else:
fake_img = fake_img_og
real_img = real_img_og
fake_pred = discriminator(fake_img)
real_pred = discriminator(real_img)
logistic_loss = d_logistic_loss(real_pred, fake_pred)
loss_dict["Discriminator"] += logistic_loss.detach()
loss_dict["Real Score"] += real_pred.mean().detach()
loss_dict["Fake Score"] += fake_pred.mean().detach()
d_loss = logistic_loss
if args.contrastive > 0:
contrast_learner(fake_img_og, fake_img, accumulate=True)
contrast_learner(real_img_og, real_img, accumulate=True)
contrast_loss = cl_module.calculate_loss()
loss_dict["Contrastive"] += contrast_loss.detach()
d_loss += args.contrastive * contrast_loss
if args.balanced_consistency > 0:
consistency_loss = mse(
real_pred, discriminator(real_img_og)) + mse(
fake_pred, discriminator(fake_img_og))
loss_dict["Consistency"] += consistency_loss.detach()
d_loss += args.balanced_consistency * consistency_loss
d_loss /= args.num_accumulate
d_loss.backward()
d_optim.step()
if args.r1 > 0 and i % args.d_reg_every == 0:
discriminator.zero_grad()
for _ in range(args.num_accumulate):
real_img = next(loader).to(device)
real_img.requires_grad = True
real_pred = discriminator(real_img)
r1_loss = d_r1_penalty(real_img, real_pred, args)
loss_dict["R1 Penalty"] += r1_loss.detach().squeeze()
r1_loss = args.r1 * args.d_reg_every * r1_loss / args.num_accumulate
r1_loss.backward()
d_optim.step()
if args.augment and args.augment_p == 0:
ada_augment += th.tensor(
(th.sign(real_pred).sum().item(), real_pred.shape[0]),
device=device)
ada_augment = reduce_sum(ada_augment)
if ada_augment[1] > 255:
pred_signs, n_pred = ada_augment.tolist()
r_t_stat = pred_signs / n_pred
loss_dict["Rt"] = th.tensor(r_t_stat, device=device).float()
if r_t_stat > args.ada_target:
sign = 1
else:
sign = -1
ada_aug_p += sign * ada_aug_step * n_pred
ada_aug_p = min(1, max(0, ada_aug_p))
ada_augment.mul_(0)
loss_dict["Augment"] = th.tensor(ada_aug_p,
device=device).float()
requires_grad(generator, True)
requires_grad(discriminator, False)
generator.zero_grad()
for _ in range(args.num_accumulate):
noise = make_noise(args.batch_size, args.latent_size,
args.mixing_prob)
fake_img, _ = generator(noise)
if args.augment:
fake_img, _ = augment(fake_img, ada_aug_p)
fake_pred = discriminator(fake_img)
g_loss = g_non_saturating_loss(fake_pred)
loss_dict["Generator"] += g_loss.detach()
g_loss /= args.num_accumulate
g_loss.backward()
g_optim.step()
if args.path_regularize > 0 and i % args.g_reg_every == 0:
generator.zero_grad()
for _ in range(args.num_accumulate):
path_loss, mean_path_length = g_path_length_regularization(
generator, mean_path_length, args)
loss_dict["Path Length Regularization"] += path_loss.detach()
path_loss = args.path_regularize * args.g_reg_every * path_loss / args.num_accumulate
path_loss.backward()
g_optim.step()
accumulate(g_ema, g_module)
loss_reduced = reduce_loss_dict(loss_dict)
log_dict = {
k: v.mean().item() / args.num_accumulate
for k, v in loss_reduced.items() if v != 0
}
if get_rank() == 0:
if args.log_spec_norm:
G_norms = []
for name, spec_norm in g_module.named_buffers():
if "spectral_norm" in name:
G_norms.append(spec_norm.cpu().numpy())
G_norms = np.array(G_norms)
D_norms = []
for name, spec_norm in d_module.named_buffers():
if "spectral_norm" in name:
D_norms.append(spec_norm.cpu().numpy())
D_norms = np.array(D_norms)
log_dict[f"Spectral Norms/G min spectral norm"] = np.log(
G_norms).min()
log_dict[f"Spectral Norms/G mean spectral norm"] = np.log(
G_norms).mean()
log_dict[f"Spectral Norms/G max spectral norm"] = np.log(
G_norms).max()
log_dict[f"Spectral Norms/D min spectral norm"] = np.log(
D_norms).min()
log_dict[f"Spectral Norms/D mean spectral norm"] = np.log(
D_norms).mean()
log_dict[f"Spectral Norms/D max spectral norm"] = np.log(
D_norms).max()
if i % args.img_every == 0:
gc.collect()
th.cuda.empty_cache()
with th.no_grad():
g_ema.eval()
sample = []
for sub in range(0, len(sample_z), args.batch_size):
subsample, _ = g_ema(
[sample_z[sub:sub + args.batch_size]])
sample.append(subsample.cpu())
sample = th.cat(sample)
grid = utils.make_grid(sample,
nrow=10,
normalize=True,
range=(-1, 1))
log_dict["Generated Images EMA"] = [
wandb.Image(grid, caption=f"Step {i}")
]
if i % args.eval_every == 0:
fid_dict = validation.fid(g_ema, args.val_batch_size,
args.fid_n_sample,
args.fid_truncation, args.name)
fid = fid_dict["FID"]
fids.append(fid)
density = fid_dict["Density"]
coverage = fid_dict["Coverage"]
ppl = validation.ppl(
g_ema,
args.val_batch_size,
args.ppl_n_sample,
args.ppl_space,
args.ppl_crop,
args.latent_size,
)
log_dict["Evaluation/FID"] = fid
log_dict["Sweep/FID_smooth"] = gaussian_filter(
np.array(fids), [5])[-1]
log_dict["Evaluation/Density"] = density
log_dict["Evaluation/Coverage"] = coverage
log_dict["Evaluation/PPL"] = ppl
gc.collect()
th.cuda.empty_cache()
wandb.log(log_dict)
description = (
f"FID: {fid:.4f} PPL: {ppl:.4f} Dens: {density:.4f} Cov: {coverage:.4f} "
+
f"G: {log_dict['Generator']:.4f} D: {log_dict['Discriminator']:.4f}"
)
if "Augment" in log_dict:
description += f" Aug: {log_dict['Augment']:.4f}" # Rt: {log_dict['Rt']:.4f}"
if "R1 Penalty" in log_dict:
description += f" R1: {log_dict['R1 Penalty']:.4f}"
if "Path Length Regularization" in log_dict:
description += f" Path: {log_dict['Path Length Regularization']:.4f}"
pbar.set_description(description)
if i % args.checkpoint_every == 0:
check_name = "-".join([
args.name,
args.runname,
wandb.run.dir.split("/")[-1].split("-")[-1],
int(fid),
args.size,
str(i).zfill(6),
])
th.save(
{
"g": g_module.state_dict(),
"d": d_module.state_dict(),
# "cl": cl_module.state_dict(),
"g_ema": g_ema.state_dict(),
"g_optim": g_optim.state_dict(),
"d_optim": d_optim.state_dict(),
},
f"/home/hans/modelzoo/maua-sg2/{check_name}.pt",
)
def main():
# tensorflow input and output
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_HEIGHT, IMAGE_WIDTH, 3],
name="input_image")
annotation = tf.placeholder(tf.int32,
shape=[None, IMAGE_HEIGHT, IMAGE_WIDTH, 1],
name="annotation")
pred_annotation, logits = inference(image, keep_probability)
# Summary
print('====================================================')
tf.summary.image("input_image", image, max_outputs=4)
tf.summary.image("ground_truth",
tf.cast(annotation * 255, tf.uint8),
max_outputs=4)
tf.summary.image("pred_annotation",
tf.cast(pred_annotation * 255, tf.uint8),
max_outputs=4)
loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits,
labels=tf.squeeze(annotation, squeeze_dims=[3]),
name="entropy")))
tf.summary.scalar("train_entropy", loss)
trainable_var = tf.trainable_variables()
if args.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
print("> [FCN] Setting up summary op...")
summary_op = tf.summary.merge_all()
# Validation summary
val_summary = tf.summary.scalar("validation_entropy", loss)
# Read data
print("> [FCN] Setting up image reader...")
train_records, valid_records = read_dataset(args.data_dir)
print('> [FCN] Train len:', len(train_records))
print('> [FCN] Val len:', len(valid_records))
t = timer.Timer() # Qhan's timer
if args.mode != 'test':
print("> [FCN] Setting up dataset reader")
image_options = {
'resize': True,
'resize_height': IMAGE_HEIGHT,
'resize_width': IMAGE_WIDTH
}
if args.mode == 'train':
t.tic()
train_dataset_reader = dataset.BatchDatset(train_records,
image_options,
mode='train')
load_time = t.toc()
print('> [FCN] Train data set loaded. %.4f ms' % (load_time))
t.tic()
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options,
mode='val')
load_time = t.toc()
print('> [FCN] Validation data set loaded. %.4f ms' % (load_time))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.90,
allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
# Initialize model
print("> [FCN] Setting up Saver...", flush=True)
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(args.logs_dir, sess.graph)
print("> [FCN] Initialize variables... ", flush=True, end='')
t.tic()
sess.run(tf.global_variables_initializer())
print('%.4f ms' % (t.toc()))
t.tic()
ckpt = tf.train.get_checkpoint_state(args.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("> [FCN] Model restored..." + ckpt.model_checkpoint_path +
', %.4f ms' % (t.toc()))
print('==================================================== [%s]' %
args.mode)
if args.mode == 'train':
np.random.seed(1028)
start = args.start_iter
end = start + args.iter + 1
for itr in range(start, end):
# Read batch data
train_images, train_annotations = train_dataset_reader.next_batch(
args.batch_size)
images = np.zeros_like(train_images)
annotations = np.zeros_like(train_annotations)
# Data augmentation
for i, (im, ann) in enumerate(zip(train_images,
train_annotations)):
flip_prob = np.random.random()
aug_type = np.random.randint(0, 3)
randoms = np.random.random(2)
images[i] = augment(im, flip_prob, aug_type, randoms)
annotations[i] = augment(ann, flip_prob, aug_type, randoms)
t.tic()
feed_dict = {
image: images,
annotation: annotations,
keep_probability: 0.85
}
sess.run(train_op, feed_dict=feed_dict)
train_time = t.toc()
if itr % 10 == 0 and itr > 10:
train_loss, summary_str = sess.run([loss, summary_op],
feed_dict=feed_dict)
summary_writer.add_summary(summary_str, itr)
print("[%6d], Train_loss: %g, %.4f ms" %
(itr, train_loss, train_time),
flush=True)
if itr % 100 == 0 and itr != 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(
args.batch_size * 2)
val_feed_dict = {
image: valid_images,
annotation: valid_annotations,
keep_probability: 1.0
}
t.tic()
val_loss, val_str = sess.run([loss, val_summary],
feed_dict=val_feed_dict)
val_time = t.toc()
summary_writer.add_summary(val_str, itr)
print("[%6d], Validation_loss: %g, %.4f ms" %
(itr, val_loss, val_time))
if itr % 1000 == 0 and itr != 0:
saver.save(sess, args.logs_dir + "model.ckpt", itr)
elif args.mode == 'visualize':
for itr in range(20):
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
1)
t.tic()
pred = sess.run(pred_annotation,
feed_dict={
image: valid_images,
keep_probability: 1.0
})
val_time = t.toc()
valid_annotations = np.squeeze(valid_annotations, axis=3)
pred = np.squeeze(pred, axis=3)
utils.save_image(valid_images[0].astype(np.uint8),
args.res_dir,
name="inp_" + str(itr))
utils.save_image(valid_annotations[0].astype(np.uint8),
args.res_dir,
name="gt_" + str(itr))
utils.save_image(pred[0].astype(np.uint8),
args.res_dir,
name="pred_" + str(itr))
print("> [FCN] Saved image: %d, %.4f ms" % (itr, val_time))
elif args.mode == 'test':
testlist = args.testlist
images, names, (H, W) = read_test_data(testlist, IMAGE_HEIGHT,
IMAGE_WIDTH)
for i, (im, name) in enumerate(zip(images, names)):
t.tic()
pred = sess.run(pred_annotation,
feed_dict={
image: im.reshape((1, ) + im.shape),
keep_probability: 1.0
})
test_time = t.toc()
pred = pred.reshape(IMAGE_HEIGHT, IMAGE_WIDTH)
if args.video:
save_video_image(im, pred,
args.res_dir + '/pred_%05d' % (i) + '.png', H,
W)
else:
misc.imsave(args.res_dir + '/inp_%d' % (i) + '.png',
im.astype(np.uint8))
misc.imsave(args.res_dir + '/pred_%d' % (i) + '.png',
pred.astype(np.uint8))
print('> [FCN] Img: %d,' % (i) + ' Name: ' + name +
', %.4f ms' % test_time)
else:
pass
from augment import augment
from keras.layers import LSTM, Input, RepeatVector
from keras.models import Model
from preprocess import scale
from scipy.spatial.distance import pdist, squareform
from sklearn.metrics import silhouette_score
from statsmodels.tsa.stattools import coint
# First party modules
import n2d
# real data for clustering
test_x = scale("Data/stock_close.csv")
# fake data for training
train_x = augment(test_x, 100)
# transpose for our autoencoder
train_x = train_x.T
# x_test = np.asarray(test_x.values)
# x_test = x_test.reshape(476, 1225, 1)
#
# train_x = train_x.reshape(47600, 1225, 1)
#
# x.shape[0]
# x.shape[1]
# not used, an experiment
# it works but i lack any of understanding of LSTMs and how to input the data so
# this will be for another example
class LSTMAE:
def get_image(filename, deterministic):
with gzip.open(filename,'rb') as f:
lung = pickle.load(f)
truth_filename = filename.replace('lung','nodule')
segmentation_filename = filename.replace('lung','lung_masks')
#segmentation_filename = re.sub(r'subset[0-9]','',segmentation_filename)
if os.path.isfile(truth_filename):
with gzip.open(truth_filename,'rb') as f:
truth = np.array(pickle.load(f),dtype=np.float32)
else:
truth = np.zeros_like(lung)
if os.path.isfile(segmentation_filename):
with gzip.open(segmentation_filename,'rb') as f:
outside = np.where(pickle.load(f)>0,0,1)
else:
outside = np.where(lung==0,1,0)
print 'lung not found'
if P.ERODE_SEGMENTATION > 0:
kernel = skimage.morphology.disk(P.ERODE_SEGMENTATION)
outside = skimage.morphology.binary_erosion(outside, kernel)
outside = np.array(outside, dtype=np.float32)
if P.AUGMENT and not deterministic:
lung, truth, outside = augment([lung, truth, outside])
if P.RANDOM_CROP > 0:
im_x = lung.shape[0]
im_y = lung.shape[1]
x = np.random.randint(0, max(1,im_x-P.RANDOM_CROP))
y = np.random.randint(0, max(1,im_y-P.RANDOM_CROP))
lung = lung[x:x+P.RANDOM_CROP, y:y+P.RANDOM_CROP]
truth = truth[x:x+P.RANDOM_CROP, y:y+P.RANDOM_CROP]
outside = outside[x:x+P.RANDOM_CROP, y:y+P.RANDOM_CROP]
truth = np.array(np.round(truth),dtype=np.int64)
outside = np.array(np.round(outside),dtype=np.int64)
#Set label of outside pixels to -10
truth = truth - (outside*10)
lung = lung*(1-outside)
lung = lung-outside*3000
if P.INPUT_SIZE > 0:
lung = crop_or_pad(lung, INPUT_SIZE, -3000)
truth = crop_or_pad(truth, OUTPUT_SIZE, 0)
outside = crop_or_pad(outside, OUTPUT_SIZE, 1)
else:
out_size = output_size_for_input(lung.shape[1], P.DEPTH)
#lung = crop_or_pad(lung, INPUT_SIZE, -1000)
truth = crop_or_pad(truth, out_size, 0)
outside = crop_or_pad(outside, out_size, 1)
lung = normalize.normalize(lung)
lung = np.expand_dims(np.expand_dims(lung, axis=0),axis=0)
if P.ZERO_CENTER:
lung = lung - P.MEAN_PIXEL
truth = np.array(np.expand_dims(np.expand_dims(truth, axis=0),axis=0),dtype=np.int64)
return lung, truth
if __name__ == '__main__':
if len(argv) != 7:
stderr.write(("USAGE: python3 %s train_file dev_file " +
"aug_factor num_epochs alpha modeloutfile\n") % argv[0])
exit(1)
TRAIN_FN = argv[1]
DEV_FN = argv[2]
AUG_FACTOR = int(argv[3])
attention.EPOCHS = int(argv[4])
ALPHA = float(argv[5])
O_FN = argv[6]
data = augment([
l.strip().split('\t')
for l in codecs.open(TRAIN_FN, 'r', "utf-8").read().split('\n')
if l.strip() != ''
], AUG_FACTOR)
idata = [[c for c in lemma] + tags.split(';') for lemma, _, tags in data]
odata = [[c for c in wf] for _, wf, _ in data]
devdata = [
l.strip().split('\t')
for l in codecs.open(DEV_FN, 'r', "utf-8").read().split('\n')
if l.strip() != ''
]
idevdata = [[c for c in lemma] + tags.split(';')
for lemma, _, tags in devdata]
odevdata = [[c for c in wf] for _, wf, _ in devdata]
characters = set([attention.EOS])
def train(args):
# Context
ctx = get_extension_context(
args.context, device_id=args.device_id, type_config=args.type_config)
nn.set_default_context(ctx)
aug_list = args.aug_list
# Model
scope_gen = "Generator"
scope_dis = "Discriminator"
# generator loss
z = nn.Variable([args.batch_size, args.latent, 1, 1])
x_fake = Generator(z, scope_name=scope_gen, img_size=args.image_size)
p_fake = Discriminator([augment(xf, aug_list)
for xf in x_fake], label="fake", scope_name=scope_dis)
lossG = loss_gen(p_fake)
# discriminator loss
x_real = nn.Variable(
[args.batch_size, 3, args.image_size, args.image_size])
x_real_aug = augment(x_real, aug_list)
p_real, rec_imgs, part = Discriminator(
x_real_aug, label="real", scope_name=scope_dis)
lossD_fake = loss_dis_fake(p_fake)
lossD_real = loss_dis_real(p_real, rec_imgs, part, x_real_aug)
lossD = lossD_fake + lossD_real
# generator with fixed latent values for test
# Use train=True even in an inference phase
z_test = nn.Variable.from_numpy_array(
np.random.randn(args.batch_size, args.latent, 1, 1))
x_test = Generator(z_test, scope_name=scope_gen,
train=True, img_size=args.image_size)[0]
# Exponential Moving Average (EMA) model
# Use train=True even in an inference phase
scope_gen_ema = "Generator_EMA"
x_test_ema = Generator(z_test, scope_name=scope_gen_ema,
train=True, img_size=args.image_size)[0]
copy_params(scope_gen, scope_gen_ema)
update_ema_var = make_ema_updater(scope_gen_ema, scope_gen, 0.999)
# Solver
solver_gen = S.Adam(args.lr, beta1=0.5)
solver_dis = S.Adam(args.lr, beta1=0.5)
with nn.parameter_scope(scope_gen):
params_gen = nn.get_parameters()
solver_gen.set_parameters(params_gen)
with nn.parameter_scope(scope_dis):
params_dis = nn.get_parameters()
solver_dis.set_parameters(params_dis)
# Monitor
monitor = Monitor(args.monitor_path)
monitor_loss_gen = MonitorSeries(
"Generator Loss", monitor, interval=10)
monitor_loss_dis_real = MonitorSeries(
"Discriminator Loss Real", monitor, interval=10)
monitor_loss_dis_fake = MonitorSeries(
"Discriminator Loss Fake", monitor, interval=10)
monitor_time = MonitorTimeElapsed(
"Training Time", monitor, interval=10)
monitor_image_tile_train = MonitorImageTile("Image Tile Train", monitor,
num_images=args.batch_size,
interval=1,
normalize_method=lambda x: (x + 1.) / 2.)
monitor_image_tile_test = MonitorImageTile("Image Tile Test", monitor,
num_images=args.batch_size,
interval=1,
normalize_method=lambda x: (x + 1.) / 2.)
monitor_image_tile_test_ema = MonitorImageTile("Image Tile Test EMA", monitor,
num_images=args.batch_size,
interval=1,
normalize_method=lambda x: (x + 1.) / 2.)
# Data Iterator
rng = np.random.RandomState(141)
di = data_iterator(args.img_path, args.batch_size,
imsize=(args.image_size, args.image_size),
num_samples=args.train_samples, rng=rng)
# Train loop
for i in range(args.max_iter):
# Train discriminator
x_fake[0].need_grad = False # no need backward to generator
x_fake[1].need_grad = False # no need backward to generator
solver_dis.zero_grad()
x_real.d = di.next()[0]
z.d = np.random.randn(args.batch_size, args.latent, 1, 1)
lossD.forward()
lossD.backward()
solver_dis.update()
# Train generator
x_fake[0].need_grad = True # need backward to generator
x_fake[1].need_grad = True # need backward to generator
solver_gen.zero_grad()
lossG.forward()
lossG.backward()
solver_gen.update()
# Update EMA model
update_ema_var.forward()
# Monitor
monitor_loss_gen.add(i, lossG.d)
monitor_loss_dis_real.add(i, lossD_real.d)
monitor_loss_dis_fake.add(i, lossD_fake.d)
monitor_time.add(i)
# Save
if (i+1) % args.save_interval == 0:
with nn.parameter_scope(scope_gen):
nn.save_parameters(os.path.join(
args.monitor_path, "Gen_iter{}.h5".format(i+1)))
with nn.parameter_scope(scope_gen_ema):
nn.save_parameters(os.path.join(
args.monitor_path, "GenEMA_iter{}.h5".format(i+1)))
with nn.parameter_scope(scope_dis):
nn.save_parameters(os.path.join(
args.monitor_path, "Dis_iter{}.h5".format(i+1)))
if (i+1) % args.test_interval == 0:
x_test.forward(clear_buffer=True)
x_test_ema.forward(clear_buffer=True)
monitor_image_tile_train.add(i+1, x_fake[0])
monitor_image_tile_test.add(i+1, x_test)
monitor_image_tile_test_ema.add(i+1, x_test_ema)
# Last
x_test.forward(clear_buffer=True)
x_test_ema.forward(clear_buffer=True)
monitor_image_tile_train.add(args.max_iter, x_fake[0])
monitor_image_tile_test.add(args.max_iter, x_test)
monitor_image_tile_test_ema.add(args.max_iter, x_test_ema)
with nn.parameter_scope(scope_gen):
nn.save_parameters(os.path.join(args.monitor_path,
"Gen_iter{}.h5".format(args.max_iter)))
with nn.parameter_scope(scope_gen_ema):
nn.save_parameters(os.path.join(args.monitor_path,
"GenEMA_iter{}.h5".format(args.max_iter)))
with nn.parameter_scope(scope_dis):
nn.save_parameters(os.path.join(args.monitor_path,
"Dis_iter{}.h5".format(args.max_iter)))
"""
Preprocessing for the T1-MRI images consists of 4 steps:
1. Loading from disk (`subject.py`) [DONE]
2. Data augmentation (`augment.py`) *
3. Data segmentation (`segment.py`) *
4. Align and normalize (`normalization.py`) * [DONE]
5. K-Fold Cross validation (`cross_validation.py`) *
* = Exports NumPy compressed (`.npz`) files.
"""
from subject import LA5C_SUBJECTS
from augment import augment
from normalization import whiten, normalize
from matplotlib import pyplot as plt
from skimage.io import imshow, imshow_collection
subjs = LA5C_SUBJECTS.select(lambda s: s['diagnosis'] in ['CONTROL', 'SCHZ'])
sample = subjs.subjects[42].load_anat(
'space-MNI152NLin2009cAsym_preproc').get_data()
print(f"Found {len(subjs)} subjects.")
subjs = augment(subjs)
# subjs = [subj.load_anat('space-MNI152NLin2009cAsym_preproc').get_data() for subj in subjs]
def train(args, model, sess, dataset):
print('|========= START TRAINING =========|')
if not os.path.isdir(args.path_summary): os.makedirs(args.path_summary)
if not os.path.isdir(args.path_model): os.makedirs(args.path_model)
saver = tf.train.Saver()
random_state = np.random.RandomState(9)
writer = {}
writer['train'] = tf.summary.FileWriter(args.path_summary + '/train',
sess.graph)
writer['val'] = tf.summary.FileWriter(args.path_summary + '/val')
t_start = time.time()
if args.train_iterations == 0:
saver.save(sess, args.path_model + '/itr-0')
for itr in range(args.train_iterations):
batch = dataset.get_next_batch('train', args.batch_size)
batch = augment(batch, args.aug_kinds, random_state)
feed_dict = {}
feed_dict.update(
{model.inputs[key]: batch[key]
for key in ['input', 'label']})
feed_dict.update({
model.compress: False,
model.is_train: True,
model.pruned: True
})
input_tensors = [model.outputs] # always execute the graph outputs
if (itr + 1) % args.check_interval == 0:
input_tensors.extend([model.summ_op, model.sparsity])
input_tensors.extend([model.train_op])
result = sess.run(input_tensors, feed_dict)
# Check on validation set.
if (itr + 1) % args.check_interval == 0:
batch = dataset.get_next_batch('val', args.batch_size)
batch = augment(batch, args.aug_kinds, random_state)
feed_dict = {}
feed_dict.update(
{model.inputs[key]: batch[key]
for key in ['input', 'label']})
feed_dict.update({
model.compress: False,
model.is_train: False,
model.pruned: True
})
input_tensors = [model.outputs, model.summ_op, model.sparsity]
result_val = sess.run(input_tensors, feed_dict)
# Check summary and print results
if (itr + 1) % args.check_interval == 0:
writer['train'].add_summary(result[1], itr)
writer['val'].add_summary(result_val[1], itr)
pstr = '(train/val) los:{:.3f}/{:.3f} acc:{:.3f}/{:.3f} spa:{:.3f}'.format(
result[0]['los'],
result_val[0]['los'],
result[0]['acc'],
result_val[0]['acc'],
result[2],
)
print('itr{}: {} (t:{:.1f})'.format(itr + 1, pstr,
time.time() - t_start))
t_start = time.time()
# Save model
if (itr + 1) % args.save_interval == 0:
saver.save(sess, args.path_model + '/itr-' + str(itr))
def data_augment(img):
# instagram
# distortion
# Assume camera is facing front and in the middle of bot.
return augment(img)
if len(argv) != 8:
stderr.write(
("USAGE: python3 %s train_file dev_file " +
"aug_factor num_epochs alpha beta modeloutfile\n") % argv[0])
exit(1)
TRAIN_FN = argv[1]
DEV_FN = argv[2]
AUG_FACTOR = int(argv[3])
attention.EPOCHS = int(argv[4])
ALPHA = float(argv[5])
BETA = float(argv[6])
O_FN = argv[7]
data = augment([
l.strip().split('\t')
for l in open(TRAIN_FN).read().split('\n') if l.strip() != ''
], AUG_FACTOR)
idata = [[c for c in lemma] + tags.split(';') for lemma, _, tags in data]
odata = [[c for c in wf] for _, wf, _ in data]
devdata = [
l.strip().split('\t') for l in open(DEV_FN).read().split('\n')
if l.strip() != ''
]
idevdata = [[c for c in lemma] + tags.split(';')
for lemma, _, tags in devdata]
odevdata = [[c for c in wf] for _, wf, _ in devdata]
characters = set([attention.EOS])
for wf in idata + odata + idevdata + odevdata:
def get_image(filename, deterministic):
# print filename
with gzip.open(filename, 'rb') as f:
lung = pickle.load(f)
np.set_printoptions(threshold='nan')
segmentation_filename = filename.replace('lung', 'lung_masks')
truth_filename = filename.replace('lung', 'nodule')
if os.path.isfile(truth_filename):
with gzip.open(truth_filename, 'rb') as f:
truth = np.array(pickle.load(f), dtype=np.float32)
else:
truth = np.zeros_like(lung)
# print np.sum(truth)
# print truth.shape
# print truth_filename
if os.path.isfile(segmentation_filename):
with gzip.open(segmentation_filename, 'rb') as f:
# print "segment file: ", segmentation_filename
outside = np.where(pickle.load(f) > 0, 0, 1)
else:
outside = np.where(lung == 0, 1, 0)
# print outside
print 'lung masks are not found'
# print "[1] truth number and outside number: ", np.sum(truth == 1), np.sum(outside)
if P.ERODE_SEGMENTATION > 0:
kernel = skimage.morphology.disk(P.ERODE_SEGMENTATION)
outside = skimage.morphology.binary_erosion(outside, kernel)
outside = np.array(outside, dtype=np.float32)
if P.AUGMENT and not deterministic:
lung, truth, outside = augment([lung, truth, outside])
if P.RANDOM_CROP > 0:
im_x = lung.shape[0]
im_y = lung.shape[1]
x = np.random.randint(0, max(1, im_x - P.RANDOM_CROP))
y = np.random.randint(0, max(1, im_y - P.RANDOM_CROP))
lung = lung[x:x + P.RANDOM_CROP, y:y + P.RANDOM_CROP]
truth = truth[x:x + P.RANDOM_CROP, y:y + P.RANDOM_CROP]
outside = outside[x:x + P.RANDOM_CROP, y:y + P.RANDOM_CROP]
truth = np.array(np.round(truth), dtype=np.int64)
outside = np.array(np.round(outside), dtype=np.int64)
# print "[2] truth number and outside number: ", np.sum(truth == 1), np.sum(outside)
# Set label of outside pixels to -10
truth = truth - (outside * 10)
lung = lung * (1 - outside)
lung = lung - outside * 3000
if P.INPUT_SIZE > 0:
lung = crop_or_pad(lung, INPUT_SIZE, -3000)
truth = crop_or_pad(truth, OUTPUT_SIZE, 0)
outside = crop_or_pad(outside, OUTPUT_SIZE, 1)
else:
out_size = output_size_for_input(lung.shape[1], P.DEPTH)
# lung = crop_or_pad(lung, INPUT_SIZE, -1000)
truth = crop_or_pad(truth, out_size, 0)
outside = crop_or_pad(outside, out_size, 1)
lung = normalize.normalize(lung)
lung = np.expand_dims(np.expand_dims(lung, axis=0), axis=0)
if P.ZERO_CENTER:
lung = lung - P.MEAN_PIXEL
if P.GAUSSIAN_NOISE > 0:
sigma = P.GAUSSIAN_NOISE
mean = 0.0
gauss = np.random.normal(mean, sigma, lung.shape)
lung = lung + gauss
truth = np.array(np.expand_dims(np.expand_dims(truth, axis=0), axis=0),
dtype=np.int64)
# print "[3] truth number and outside number: ", np.sum(truth == 1), np.sum(outside)
return lung, truth
def mobilenet_v1_model_fn(features, labels, mode, params = {'dim':[512,768,3], 'format':"NHWC", 'num_classes':257, 'learning_rate':1e-4}):
if params['format'] == "NHWC":
iHeight = 0
iWidth = 1
batchnorm_axis = 3
else:
raise ValueError('mobilenet_v1_model_fn format {} not supported'.format(params['format']))
def parser(record):
label_key = "image/label"
bytes_key = "image/encoded"
parsed = tf.parse_single_example(record, {
bytes_key : tf.FixedLenFeature([], tf.string),
label_key : tf.FixedLenFeature([], tf.int64),
})
image = tf.decode_raw(parsed[bytes_key], tf.uint8)
dims = [width, height, channels]
image = tf.reshape(image, dims)
return { "image" : image }, parsed[label_key]
images = features["image"]
images = tf.image.convert_image_dtype(images, tf.float32)
# Test support and performance of NHWC and NHWC
#if params.data_format == 'NHWC':
# Convert the inputs from channels_last (NHWC) to channels_first (NHWC).
# This provides a large performance boost on GPU. See
# https://www.tensorflow.org/performance/performance_guide#data_formats
#images = tf.transpose(images, [0, 2, 1, 3], name="nhwc_input")
# Augment dataset when training
if mode == tf.contrib.learn.ModeKeys.TRAIN:
images = augment(images, params)
images = tf.image.per_image_standardization(images)
images = tf.image.resize_with_crop_or_pad(images,params['dim'][iHeight],params['dim'][iWidth])
# make_mobilenet_v1
logits = make_mobilenet_v1(images, mode, batchnorm_axis, params['format'], params['num_classes'])
predictions = {
# Returns the highest prediction from the output of logits.
tf.contrib.learn.PredictionKey.CLASSES : tf.argmax(logits, axis=1, name="predicted_classes"),
# Softmax squashes arbitrary real values into a range of 0 to 1 and returns the
# probabilities of each of the classes. Name is used for logging.
tf.contrib.learn.PredictionKey.PROBABILITIES : tf.nn.softmax(logits, name="predicted_probability")
}
# Run-time prediction
if mode == tf.estimator.ModeKeys.PREDICT:
exports = {'predictions' : tf.estimator.export.PredictOutput(predictions)}
return tf.estimator.EstimatorSpec(mode, predictions=predictions, export_outputs=exports)
#labels = tf.squeeze(labels, axis=3) # reduce the channel dimension.
predicted_labels = predictions[tf.contrib.learn.PredictionKey.CLASSES]
model_metrics = metrics(labels, predicted_labels, tf.contrib.learn.PredictionKey.CLASSES)
tf.summary.scalar("accuracy", model_metrics['accuracy'][1])
tf.summary.scalar("precision", model_metrics['precision'][1])
tf.summary.scalar("recall", model_metrics['recall'][1])
#tf.summary.scalar("meaniou", model_metrics['meaniou'][1])
logits_by_num_classes = tf.reshape(logits, [-1, params['num_classes']])
labels_flat = tf.reshape(labels, [-1, ])
valid_indices = tf.to_int32(labels_flat <= params['num_classes'] - 1)
valid_logits = tf.dynamic_partition(logits_by_num_classes, valid_indices, num_partitions=2)[1]
valid_labels = tf.dynamic_partition(labels_flat, valid_indices, num_partitions=2)[1]
pred_loss = loss(valid_logits, valid_labels, params['num_classes'])
# Training time evaluation
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=pred_loss, eval_metric_ops=model_metrics)
# Train the model with AdamOptimizer
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer(learning_rate=params['learning_rate'])
train_op = optimizer.minimize(pred_loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=pred_loss, train_op=train_op)
def do_step(chosen_tasks, training):
losses = []
accuracies_c = []
accuracies_d = []
gradient_accum = None
for t_id, t in enumerate(chosen_tasks):
corrects_c = []
corrects_d = []
with tf.GradientTape() as tape:
if training and prm['AUGMENT']:
t = augment(t, prm)
target_shape = t[OUT_TEST].shape
target_grid = t[OUT_TEST]
result_c, result_d = tasksolver(t)
if prm['SIZEPREDTYPE'] == SizePredType.CLASSIFICATION:
loss_c = tf.reduce_mean(
tf.keras.losses.sparse_categorical_crossentropy(
target_shape, result_c, from_logits=True))
pred_ids = tf.keras.backend.argmax(result_c, axis=-1)
elif prm['SIZEPREDTYPE'] == SizePredType.REAL_NUMBER:
result_c = tf.squeeze(result_c)
loss_c = tf.reduce_mean(
tf.square(tf.cast(target_shape, tf.float32) - result_c))
pred_ids = tf.cast(tf.math.round(result_c), tf.int64)
correct_c = tf.reduce_all(tf.equal(pred_ids, target_shape))
pred_grid = tf.keras.backend.argmax(result_d, axis=-1)
if correct_c.numpy() == True:
pred_grid = tf.reshape(
pred_grid, target_grid.shape) #yes i'm doing it twice.
correct_d = tf.reduce_all(
tf.equal(pred_grid, tf.cast(target_grid, tf.int64)))
else:
correct_d = tf.constant(False)
if training:
pred_grid = tf.reshape(
pred_grid, target_grid.shape) #yes i'm doing it twice.
loss_d = tf.reduce_mean(
tf.keras.losses.sparse_categorical_crossentropy(
target_grid, result_d, from_logits=True))
else:
loss_d = tf.constant(0.0)
corrects_c.append(correct_c)
corrects_d.append(correct_d)
#print(pred_ids.numpy(), target_shape, correct.numpy())
losses.append(loss_c)
losses.append(loss_d)
if training:
gradients = tape.gradient(
losses,
tasksolver.trainable_variables,
unconnected_gradients=tf.UnconnectedGradients.ZERO)
if gradient_accum is None:
gradient_accum = [
tf.convert_to_tensor(tens) for tens in gradients
]
else:
for t_id2 in range(0, len(gradient_accum)):
gradient_accum[t_id2] = tf.add(
gradient_accum[t_id2],
tf.convert_to_tensor(gradients[t_id2]))
corrects_c = tf.reduce_all(tf.stack(corrects_c, axis=0))
corrects_d = tf.reduce_all(tf.stack(corrects_d, axis=0))
accuracy_c = tf.squeeze(
tf.reduce_mean(tf.cast(corrects_c, dtype=tf.float32)))
accuracy_d = tf.squeeze(
tf.reduce_mean(tf.cast(corrects_d, dtype=tf.float32)))
accuracies_c.append(accuracy_c)
accuracies_d.append(accuracy_d)
print(f"{t_id}/{len(chosen_tasks)}", end=" \r")
totalparams = tf.reduce_sum(
[tf.size(tens) for tens in tasksolver.trainable_variables])
if training:
#print(f" #gt {len(gradient_accum)} #p {totalparams} ", end='')
optimizer.apply_gradients(
zip(gradient_accum, tasksolver.trainable_variables))
return losses, accuracies_c, accuracies_d
exit(1)
TRAIN_FN = argv[1]
DEV_FN = argv[2]
MAX_SIZE = int(argv[3])
attention_task2.EPOCHS = int(argv[4])
ALPHA = float(argv[5])
BETA = float(argv[6])
O_FN = argv[7]
orig_data = [
l.strip().split('\t') for l in open(TRAIN_FN).read().split('\n')
if l.strip() != ''
]
AUG_FACTOR = max(1, int(MAX_SIZE / len(orig_data)))
data = augment(orig_data, AUG_FACTOR)
stdout.flush()
idata = [[c for c in lemma] + tags.split(';') for lemma, _, tags in data]
odata = [[c for c in wf] for _, wf, _ in data]
devdata = [
l.strip().split('\t') for l in open(DEV_FN).read().split('\n')
if l.strip() != ''
]
idevdata = [[c for c in lemma] + tags.split(';')
for lemma, _, tags in devdata]
odevdata = [[c for c in wf] for _, wf, _ in devdata]
characters = set([attention_task2.EOS])
for wf in idata + odata:
