def test(score_path, data_path):
model = DrBC()
model.cuda()
load(model, MODEL_PATH)
model = model.cpu()
test_graph = TestData(score_path, data_path)
val(model, test_graph, cuda=False)
def read_song_id_and_find_genre(self, path_of_track_file, genres):
tracks = utils.load(path_of_track_file)
for gen in genres:
all_ids = []
for id, genre in tracks['track', "genre_top"].items():
if(genre == gen):
all_ids.append('{:0>6}'.format(id))
self.write_json_for_genres_all_song_ids(all_ids, gen)
def product_pipeline(ci):
job, job_folder = ci.new_job()
job_folder += "_product"
cache_folder = os.path.join(job_folder, "cache")
os.makedirs(cache_folder, exist_ok=True)
with setenv("CONAN_USER_HOME", cache_folder):
ci.run("conan config set general.revisions_enabled=True")
ci.run(
"conan config set general.default_package_id_mode=recipe_revision_mode"
)
ci.run("conan remote remove conan-center")
ci.run(
"conan remote add master http://localhost:8081/artifactory/api/conan/ci-master -f"
)
ci.run("conan user admin -p=password -r=master")
with chdir(job_folder):
ci.run("conan graph lock app/[~1.0]@user/testing --build"
) # Always all build, as soon as we export, it will change
lockfile = load("conan.lock")
ci.run(
"conan graph build-order . --build=missing --json=build-order.json"
)
build_order = json.loads(load("build-order.json"))
print("*********** BUILD-ORDER *******************\n%s" %
build_order)
for level_to_build in build_order:
for to_build in level_to_build:
ref = to_build[1].split(":")[0]
print("Building ", ref)
ci.run("conan inspect %s -a=scm --json=scm.json" % ref)
scm = json.loads(load("scm.json"))["scm"]
os.remove("scm.json")
url = scm["url"]
revision = scm["revision"]
package_pipeline(ci,
url,
branch=revision,
lockfile=lockfile,
upload="master")
ci.run(
"conan install app/[~1.0]@user/testing --lockfile -g=deploy")
ci.run(r".\app\bin\main_app.exe")
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def eval_model(cfg, models_path):
#Setup
model = ResNetLSTM(**cfg.model).cuda()
optimizer = torch.optim.SGD(model.parameters(), **cfg.optim)
epoch = load(models_path, model, optimizer, train=False)
criterion = nn.CrossEntropyLoss()
reshape_transform = transforms.Compose([transforms.ToPILImage(),
transforms.Resize((cfg.img_size, cfg.img_size)),
#transforms.Grayscale(),
transforms.ToTensor()])
base_dir = "../../../datasets/20bn-sth-sth-v2"
val_data = SthSthDataset(labels_file = "78-classes_validation.json",
transform = reshape_transform,
base_dir = base_dir,
n_frames = cfg.dataset.n_frames, #frames to pick from each video
str2id_file = cfg.dataset.str2id_file)
data_loader = torch.utils.data.DataLoader(val_data, num_workers = 2, batch_size=8 , shuffle=False)
mean_loss, top1acc, top5acc = validation(data_loader, model, criterion)
def test_model(cfg, models_path):
#Setup
model = ResNetLSTM(**cfg.model).cuda()
optimizer = torch.optim.SGD(model.parameters(), **cfg.optim) #cfg.lr
epoch = load(models_path, model, optimizer, train=False)
criterion = nn.CrossEntropyLoss()
reshape_transform = transforms.Compose([transforms.ToPILImage(),
transforms.Resize((64, 64)),
#transforms.Grayscale(),
transforms.ToTensor()])
base_dir = "../../../datasets/20bn-sth-sth-v2"
data = SthSthTestset(ids_file = "something-something-v2-test.json",
base_dir = base_dir,
transform = reshape_transform,
n_frames = 8, #frames to pick from each video
)
data_loader = torch.utils.data.DataLoader(data, num_workers = 2, batch_size=1 ,shuffle=False)
predictions = make_predictions(data_loader, model, base_dir)
#write file
output_file = "./predictions.json"
with open(output_file, 'w') as file:
file.write(json.dumps(predictions, indent=2))# use `json.loads` to do the reverse
import os
from utils.sanity_checks import sample_vectors_expected
from utils.utils import load
import numpy as np
from pprint import pprint
# tester code for checking the output vectors generated by run.py.
sample_vectors_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"sample_vectors.json")
if not os.path.isfile(sample_vectors_path):
raise Exception('Excecute run.py file before excecuting this file')
sample_vectors_actual = load(sample_vectors_path)
test_words = ["female", "cool"]
for word in test_words:
print("Your output")
pprint(sample_vectors_actual[word])
print("\nExpected output")
pprint(sample_vectors_expected[word])
print(
"\nAre the vectors similar: ",
np.allclose(sample_vectors_actual[word],
sample_vectors_expected[word],
rtol=1e-3))
print("\n" * 3)
def train(args,
generator: Generator,
discriminator: Discriminator,
feature_extractor: FeatureExtractor,
photo_dataloader,
edge_smooth_dataloader,
animation_dataloader,
checkpoint_dir=None):
tb_writter = SummaryWriter()
gen_criterion = nn.BCELoss().to(args.device)
disc_criterion = nn.BCELoss().to(args.device)
content_criterion = nn.L1Loss().to(args.device)
gen_optimizer = torch.optim.Adam(generator.parameters(),
lr=args.lr,
betas=(args.adam_beta, 0.999))
disc_optimizer = torch.optim.Adam(discriminator.parameters(),
lr=args.lr,
betas=(args.adam_beta, 0.999))
global_step = 0
global_init_step = 0
# The number of steps to skip when loading a checkpoint
skipped_step = 0
skipped_init_step = 0
cur_epoch = 0
cur_init_epoch = 0
data_len = min(len(photo_dataloader), len(edge_smooth_dataloader),
len(animation_dataloader))
if checkpoint_dir:
try:
checkpoint_dict = load(checkpoint_dir)
generator.load_state_dict(checkpoint_dict['generator'])
discriminator.load_state_dict(checkpoint_dict['discriminator'])
gen_optimizer.load_state_dict(checkpoint_dict['gen_optimizer'])
disc_optimizer.load_state_dict(checkpoint_dict['disc_optimizer'])
global_step = checkpoint_dict['global_step']
global_init_step = checkpoint_dict['global_init_step']
cur_epoch = global_step // data_len
cur_init_epoch = global_init_step // len(photo_dataloader)
skipped_step = global_step % data_len
skipped_init_step = global_init_step % len(photo_dataloader)
logger.info("Start training with,")
logger.info("In initialization step, epoch: %d, step: %d",
cur_init_epoch, skipped_init_step)
logger.info("In main train step, epoch: %d, step: %d", cur_epoch,
skipped_step)
except:
logger.info("Wrong checkpoint path")
t_total = data_len * args.n_epochs
t_init_total = len(photo_dataloader) * args.n_init_epoch
# Train!
logger.info("***** Running training *****")
logger.info(" Num photo examples = %d", len(photo_dataloader))
logger.info(" Num edge_smooth examples = %d", len(edge_smooth_dataloader))
logger.info(" Num animation examples = %d", len(animation_dataloader))
logger.info(" Num Epochs = %d", args.n_epochs)
logger.info(" Total train batch size = %d", args.batch_size)
logger.info(" Total optimization steps = %d", t_total)
logger.info(" Num Init Epochs = %d", args.n_init_epoch)
logger.info(" Total Init optimization steps = %d", t_init_total)
logger.info(" Logging steps = %d", args.logging_steps)
logger.info(" Save steps = %d", args.save_steps)
init_phase = True
try:
generator.train()
discriminator.train()
gloabl_init_loss = 0
# --- Initialization Content loss
mb = master_bar(range(cur_init_epoch, args.n_init_epoch))
for init_epoch in mb:
epoch_iter = progress_bar(photo_dataloader, parent=mb)
for step, (photo, _) in enumerate(epoch_iter):
if skipped_init_step > 0:
skipped_init_step = -1
continue
photo = photo.to(args.device)
gen_optimizer.zero_grad()
x_features = feature_extractor((photo + 1) / 2).detach()
Gx = generator(photo)
Gx_features = feature_extractor((Gx + 1) / 2)
content_loss = args.content_loss_weight * content_criterion(
Gx_features, x_features)
content_loss.backward()
gen_optimizer.step()
gloabl_init_loss += content_loss.item()
global_init_step += 1
if args.save_steps > 0 and global_init_step % args.save_steps == 0:
logger.info(
"Save Initialization Phase, init_epoch: %d, init_step: %d",
init_epoch, global_init_step)
save(checkpoint_dir, global_step, global_init_step,
generator, discriminator, gen_optimizer,
disc_optimizer)
if args.logging_steps > 0 and global_init_step % args.logging_steps == 0:
tb_writter.add_scalar('Initialization Phase/Content Loss',
content_loss.item(),
global_init_step)
tb_writter.add_scalar(
'Initialization Phase/Global Generator Loss',
gloabl_init_loss / global_init_step, global_init_step)
logger.info(
"Initialization Phase, Epoch: %d, Global Step: %d, Content Loss: %.4f",
init_epoch, global_init_step,
gloabl_init_loss / (global_init_step))
# -----------------------------------------------------
logger.info("Finish Initialization Phase, save model...")
save(checkpoint_dir, global_step, global_init_step, generator,
discriminator, gen_optimizer, disc_optimizer)
init_phase = False
global_loss_D = 0
global_loss_G = 0
global_loss_content = 0
mb = master_bar(range(cur_epoch, args.n_epochs))
for epoch in mb:
epoch_iter = progress_bar(list(
zip(animation_dataloader, edge_smooth_dataloader,
photo_dataloader)),
parent=mb)
for step, ((animation, _), (edge_smoothed, _),
(photo, _)) in enumerate(epoch_iter):
if skipped_step > 0:
skipped_step = -1
continue
animation = animation.to(args.device)
edge_smoothed = edge_smoothed.to(args.device)
photo = photo.to(args.device)
disc_optimizer.zero_grad()
# --- Train discriminator
# ------ Train Discriminator with animation image
animation_disc = discriminator(animation)
animation_target = torch.ones_like(animation_disc)
loss_animation_disc = disc_criterion(animation_disc,
animation_target)
# ------ Train Discriminator with edge image
edge_smoothed_disc = discriminator(edge_smoothed)
edge_smoothed_target = torch.zeros_like(edge_smoothed_disc)
loss_edge_disc = disc_criterion(edge_smoothed_disc,
edge_smoothed_target)
# ------ Train Discriminator with generated image
generated_image = generator(photo).detach()
generated_image_disc = discriminator(generated_image)
generated_image_target = torch.zeros_like(generated_image_disc)
loss_generated_disc = disc_criterion(generated_image_disc,
generated_image_target)
loss_disc = loss_animation_disc + loss_edge_disc + loss_generated_disc
loss_disc.backward()
disc_optimizer.step()
global_loss_D += loss_disc.item()
# --- Train Generator
gen_optimizer.zero_grad()
generated_image = generator(photo)
generated_image_disc = discriminator(generated_image)
generated_image_target = torch.ones_like(generated_image_disc)
loss_adv = gen_criterion(generated_image_disc,
generated_image_target)
# ------ Train Generator with content loss
x_features = feature_extractor((photo + 1) / 2).detach()
Gx_features = feature_extractor((generated_image + 1) / 2)
loss_content = args.content_loss_weight * content_criterion(
Gx_features, x_features)
loss_gen = loss_adv + loss_content
loss_gen.backward()
gen_optimizer.step()
global_loss_G += loss_adv.item()
global_loss_content += loss_content.item()
global_step += 1
if args.save_steps > 0 and global_step % args.save_steps == 0:
logger.info("Save Training Phase, epoch: %d, step: %d",
epoch, global_step)
save(checkpoint_dir, global_step, global_init_step,
generator, discriminator, gen_optimizer,
disc_optimizer)
if args.logging_steps > 0 and global_init_step % args.logging_steps == 0:
tb_writter.add_scalar('Train Phase/Generator Loss',
loss_adv.item(), global_step)
tb_writter.add_scalar('Train Phase/Discriminator Loss',
loss_disc.item(), global_step)
tb_writter.add_scalar('Train Phase/Content Loss',
loss_content.item(), global_step)
tb_writter.add_scalar('Train Phase/Global Generator Loss',
global_loss_G / global_step,
global_step)
tb_writter.add_scalar(
'Train Phase/Global Discriminator Loss',
global_loss_D / global_step, global_step)
tb_writter.add_scalar('Train Phase/Global Content Loss',
global_loss_content / global_step,
global_step)
logger.info(
"Training Phase, Epoch: %d, Global Step: %d, Disc Loss %.4f, Gen Loss %.4f, Content Loss: %.4f",
epoch, global_step, global_loss_D / global_step,
global_loss_G / global_step,
global_loss_content / global_step)
except KeyboardInterrupt:
if init_phase:
logger.info("KeyboardInterrupt in Initialization Phase!")
logger.info("Save models, init_epoch: %d, init_step: %d",
init_epoch, global_init_step)
else:
logger.info("KeyboardInterrupt in Training Phase!")
logger.info("Save models, epoch: %d, step: %d", epoch, global_step)
save(checkpoint_dir, global_step, global_init_step, generator,
discriminator, gen_optimizer, disc_optimizer)
#!/usr/bin/env python
import sys
from numpy import *
from pylab import *
from utils.utils import load
from utils.interpolate import spline
au2ev=27.2114
c=137.036
data = load('cu-phthalo.dat')
freqshift=10.4
freq=data[:,0]
alpha_cat=(data[:,1])/3
alpha_rad=(data[:,2])/3
alpha_ani=(data[:,3])/3
sigma_cat=4*pi/c*freq*alpha_cat
sigma_rad=4*pi/c*freq*alpha_rad
sigma_ani=4*pi/c*freq*alpha_ani
freq=freq*au2ev+freqshift
x1=284.2; x2=5.6
y1=0; y2=7.4
figure(1, figsize=(8,12))
#========================================================================
# Which variables to load.
restore_var = tf.global_variables()
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if RESTORE_FROM is not None:
if load(loader, sess, RESTORE_FROM):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
DATA_TYPE = ['png','PNG','jpg','JPG']
print("[INFO] loading edge detector...")
protoPath = os.path.sep.join(["hed_model",
"deploy.prototxt"])
modelPath = os.path.sep.join(["hed_model",
"hed_pretrained_bsds.caffemodel"])
net = cv2.dnn.readNetFromCaffe(protoPath, modelPath)
def train():
from model.EDSR import EDSR
device = torch.device(args.cuda if torch.cuda.is_available() else "cpu")
transform_ = transforms.Compose([transforms.ToTensor()])
data = DatasetManager(dataset='DIV2K',
transform=transform_,
psize=[args.patch_size, args.patch_size])
dataMan = DataLoader(data, batch_size=1)
criterion = Loss.Loss(device,
patch_size=args.patch_size * 2,
losses=args.loss,
k=args.k,
out=args.o,
depth=args.depth)
EDSR = EDSR(args.scale, args.res_length)
EDSR.to(device)
trainable = filter(lambda x: x.requires_grad, EDSR.parameters())
optimizer = optim.Adam(trainable, lr=args.lr, weight_decay=200)
EDSR, criterion, optimizer, epoch, runningavg = load(
EDSR,
criterion,
optimizer,
load=args.load,
path='./checkpoints/' + args.version)
for epoch in range(epoch, args.epochs):
running_loss = 0.0
sep_losses = [0 for _ in range(len(args.loss.split(',')))]
for i, (X, y) in enumerate(dataMan, 0):
patchesX = data.extract_patches(X, batch_first=True)
patchesY = data.extract_patches(
y,
size=[data.psize[0] * 2, data.psize[1] * 2],
batch_first=True)
if patchesX.size(1) == patchesY.size(1):
patchManager = DataLoader(ImageManager(patchesX[0],
patchesY[0]),
batch_size=args.batch)
for j, (px, py) in enumerate(patchManager, 0):
optimizer.zero_grad()
outputs = EDSR(px.to(device))
loss = criterion.forward(outputs.to(device), py.to(device))
loss.backward()
optimizer.step()
running_loss += loss.item()
sep_losses = [
sepl_new.item() + sepl_old for sepl_new, sepl_old in
zip(criterion.losses, sep_losses)
]
sep_losses = [sepl / patchesY.size(1) for sepl in sep_losses]
running_loss /= patchesY.size(1)
runningavg['LOSS'].append(running_loss)
for idx, loss in enumerate(args.loss.split(','), 0):
runningavg[loss].append(sep_losses[idx])
log(log_type(running_loss, sep_losses, runningavg, epoch, i),
'./logs/' + args.version + '/loss_logs.txt')
if i % 20 == 0:
os.makedirs(os.path.dirname('./out/' + args.version + '/'),
exist_ok=True)
torchvision.utils.save_image(
outputs[-1], './out/' + args.version +
'/{}_{}_pred.png'.format(epoch + 1, i + 1))
torchvision.utils.save_image(
patchesY[0][-1], './out/' + args.version +
'/{}_{}_hr.png'.format(epoch + 1, i + 1))
torchvision.utils.save_image(
patchesX[0][-1], './out/' + args.version +
'/{}_{}_lr.png'.format(epoch + 1, i + 1))
sep_losses = [0, 0, 0]
running_loss = 0.0
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': EDSR.state_dict(),
'optimizer': optimizer.state_dict(),
'loss': criterion.state_dict(),
'ema': runningavg
},
filename='./checkpoints/' + args.version + '/Epoch_' +
args.version + '_{}.pt'.format(epoch + 1))
import os
from utils.sanity_checks import sample_vectors_expected
from utils.utils import load
import numpy as np
from pprint import pprint
# tester code for checking the output vectors generated by run.py.
sampleVectorsPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sampleVectors.json")
if not os.path.isfile(sampleVectorsPath):
raise Exception('Excecute run.py file before excecuting this file')
sample_vectors_actual = load(sampleVectorsPath)
test_words = ["female", "cool"]
for word in test_words:
print("Your output")
pprint(sample_vectors_actual[word])
print("\nExpected output")
pprint(sample_vectors_expected[word])
print("\nAre the vectors similar: ",
np.allclose(sample_vectors_actual[word], sample_vectors_expected[word], rtol=1e-3))
print("\n"*3)
def main():
"""Create the model and start the evaluation process."""
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(DATA_DIR, LIST_PATH, DATA_ID_LIST, None, False,
False, False, coord)
image, label, edge_gt = reader.image, reader.label, reader.edge
image_rev = tf.reverse(image, tf.stack([1]))
image_list = reader.image_list
image_batch = tf.stack([image, image_rev])
label_batch = tf.expand_dims(label, dim=0) # Add one batch dimension.
edge_gt_batch = tf.expand_dims(edge_gt, dim=0)
h_orig, w_orig = tf.to_float(tf.shape(image_batch)[1]), tf.to_float(
tf.shape(image_batch)[2])
image_batch050 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 0.50)),
tf.to_int32(tf.multiply(w_orig, 0.50))
]))
image_batch075 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 0.75)),
tf.to_int32(tf.multiply(w_orig, 0.75))
]))
image_batch125 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 1.25)),
tf.to_int32(tf.multiply(w_orig, 1.25))
]))
image_batch150 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 1.50)),
tf.to_int32(tf.multiply(w_orig, 1.50))
]))
image_batch175 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 1.75)),
tf.to_int32(tf.multiply(w_orig, 1.75))
]))
# Create network.
with tf.variable_scope('', reuse=False):
net_100 = PGNModel({'data': image_batch},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_050 = PGNModel({'data': image_batch050},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_075 = PGNModel({'data': image_batch075},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_125 = PGNModel({'data': image_batch125},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_150 = PGNModel({'data': image_batch150},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_175 = PGNModel({'data': image_batch175},
is_training=False,
n_classes=N_CLASSES)
# parsing net
parsing_out1_050 = net_050.layers['parsing_fc']
parsing_out1_075 = net_075.layers['parsing_fc']
parsing_out1_100 = net_100.layers['parsing_fc']
parsing_out1_125 = net_125.layers['parsing_fc']
parsing_out1_150 = net_150.layers['parsing_fc']
parsing_out1_175 = net_175.layers['parsing_fc']
parsing_out2_050 = net_050.layers['parsing_rf_fc']
parsing_out2_075 = net_075.layers['parsing_rf_fc']
parsing_out2_100 = net_100.layers['parsing_rf_fc']
parsing_out2_125 = net_125.layers['parsing_rf_fc']
parsing_out2_150 = net_150.layers['parsing_rf_fc']
parsing_out2_175 = net_175.layers['parsing_rf_fc']
# edge net
edge_out2_100 = net_100.layers['edge_rf_fc']
edge_out2_125 = net_125.layers['edge_rf_fc']
edge_out2_150 = net_150.layers['edge_rf_fc']
edge_out2_175 = net_175.layers['edge_rf_fc']
# combine resize
parsing_out1 = tf.reduce_mean(tf.stack([
tf.image.resize_images(parsing_out1_050,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_075,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_100,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_125,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_150,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_175,
tf.shape(image_batch)[1:3, ])
]),
axis=0)
parsing_out2 = tf.reduce_mean(tf.stack([
tf.image.resize_images(parsing_out2_050,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_075,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_100,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_125,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_150,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_175,
tf.shape(image_batch)[1:3, ])
]),
axis=0)
edge_out2_100 = tf.image.resize_images(edge_out2_100,
tf.shape(image_batch)[1:3, ])
edge_out2_125 = tf.image.resize_images(edge_out2_125,
tf.shape(image_batch)[1:3, ])
edge_out2_150 = tf.image.resize_images(edge_out2_150,
tf.shape(image_batch)[1:3, ])
edge_out2_175 = tf.image.resize_images(edge_out2_175,
tf.shape(image_batch)[1:3, ])
edge_out2 = tf.reduce_mean(tf.stack(
[edge_out2_100, edge_out2_125, edge_out2_150, edge_out2_175]),
axis=0)
raw_output = tf.reduce_mean(tf.stack([parsing_out1, parsing_out2]), axis=0)
head_output, tail_output = tf.unstack(raw_output, num=2, axis=0)
tail_list = tf.unstack(tail_output, num=20, axis=2)
tail_list_rev = [None] * 20
for xx in xrange(14):
tail_list_rev[xx] = tail_list[xx]
tail_list_rev[14] = tail_list[15]
tail_list_rev[15] = tail_list[14]
tail_list_rev[16] = tail_list[17]
tail_list_rev[17] = tail_list[16]
tail_list_rev[18] = tail_list[19]
tail_list_rev[19] = tail_list[18]
tail_output_rev = tf.stack(tail_list_rev, axis=2)
tail_output_rev = tf.reverse(tail_output_rev, tf.stack([1]))
raw_output_all = tf.reduce_mean(tf.stack([head_output, tail_output_rev]),
axis=0)
raw_output_all = tf.expand_dims(raw_output_all, dim=0)
pred_scores = tf.reduce_max(raw_output_all, axis=3)
raw_output_all = tf.argmax(raw_output_all, axis=3)
pred_all = tf.expand_dims(raw_output_all, dim=3) # Create 4-d tensor.
raw_edge = tf.reduce_mean(tf.stack([edge_out2]), axis=0)
head_output, tail_output = tf.unstack(raw_edge, num=2, axis=0)
tail_output_rev = tf.reverse(tail_output, tf.stack([1]))
raw_edge_all = tf.reduce_mean(tf.stack([head_output, tail_output_rev]),
axis=0)
raw_edge_all = tf.expand_dims(raw_edge_all, dim=0)
pred_edge = tf.sigmoid(raw_edge_all)
res_edge = tf.cast(tf.greater(pred_edge, 0.5), tf.int32)
# prepare ground truth
preds = tf.reshape(pred_all, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
weights = tf.cast(
tf.less_equal(gt, N_CLASSES - 1),
tf.int32) # Ignoring all labels greater than or equal to n_classes.
mIoU, update_op_iou = tf.contrib.metrics.streaming_mean_iou(
preds, gt, num_classes=N_CLASSES, weights=weights)
macc, update_op_acc = tf.contrib.metrics.streaming_accuracy(
preds, gt, weights=weights)
# precision and recall
recall, update_op_recall = tf.contrib.metrics.streaming_recall(
res_edge, edge_gt_batch)
precision, update_op_precision = tf.contrib.metrics.streaming_precision(
res_edge, edge_gt_batch)
update_op = tf.group(update_op_iou, update_op_acc, update_op_recall,
update_op_precision)
# Which variables to load.
restore_var = tf.global_variables()
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if RESTORE_FROM is not None:
if load(loader, sess, RESTORE_FROM):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# evaluate prosessing
parsing_dir = 'dataset/parse_cihp'
if os.path.exists(parsing_dir):
shutil.rmtree(parsing_dir)
if not os.path.exists(parsing_dir):
os.makedirs(parsing_dir)
# Iterate over training steps.
for step in range(NUM_STEPS):
parsing_, scores, edge_, _ = sess.run(
[pred_all, pred_scores, pred_edge, update_op])
if step % 100 == 0:
print('step {:d}'.format(step))
print(image_list[step])
img_split = image_list[step].split('/')
img_id = img_split[-1][:-4]
msk = decode_labels(parsing_, num_classes=N_CLASSES)
parsing_im = Image.fromarray(msk[0])
parsing_im.save('dataset/parse_cihp/person_vis.png')
im = Image.open('dataset/parse_cihp/person_vis.png')
new_width = 192
new_height = 256
im = im.resize((new_width, new_height), Image.ANTIALIAS)
im.save('dataset/parse_cihp/person_vis.png')
cv2.imwrite('dataset/parse_cihp/person.png', parsing_[0, :, :, 0])
im = Image.open('dataset/parse_cihp/person.png')
im = im.resize((new_width, new_height), Image.ANTIALIAS)
im.save('dataset/parse_cihp/person.png')
#sio.savemat('{}/{}.mat'.format(parsing_dir, img_id), {'data': scores[0,:,:]})
#cv2.imwrite('dataset/cloth_mask/person_mask.png', edge_[0,:,:,0] * 255)
res_mIou = mIoU.eval(session=sess)
res_macc = macc.eval(session=sess)
res_recall = recall.eval(session=sess)
res_precision = precision.eval(session=sess)
f1 = 2 * res_precision * res_recall / (res_precision + res_recall)
print('Mean IoU: {:.4f}, Mean Acc: {:.4f}'.format(res_mIou, res_macc))
print('Recall: {:.4f}, Precision: {:.4f}, F1 score: {:.4f}'.format(
res_recall, res_precision, f1))
coord.request_stop()
coord.join(threads)
def main(args):
# loading configurations
with open(args.config) as f:
config = yaml.safe_load(f)["configuration"]
name = config["Name"]
# Construct or load embeddings
print("Initializing embeddings ...")
vocab_size = config["embeddings"]["vocab_size"]
embed_size = config["embeddings"]["embed_size"]
per_num = config["embeddings"]["person_num"]
per_embed_size = config["embeddings"]["person_embed_size"]
embeddings = init_embeddings(vocab_size, embed_size, name=name)
print("\tDone.")
# Build the model and compute losses
source_ids = tf.placeholder(tf.int32, [None, 40], name="source")
target_ids = tf.placeholder(tf.int32, [None, 40], name="target")
person_ids = tf.placeholder(tf.int32, [None], name="person_ids")
lexicons_ids = tf.placeholder(tf.int32, [per_num, 1000],
name="lexicons_ids")
spectrogram = tf.placeholder(tf.float32, [None, 400, 200], name="audio")
sequence_mask = tf.placeholder(tf.bool, [None, 40], name="mask")
choice_qs = tf.placeholder(tf.float32, [None, 40], name="choice")
emo_cat = tf.placeholder(tf.int32, [None], name="emotion_category")
is_train = tf.placeholder(tf.bool)
(enc_num_layers, enc_num_units, enc_cell_type, enc_bidir, dec_num_layers,
dec_num_units, dec_cell_type, state_pass, num_emo, emo_cat_units,
emo_int_units, infer_batch_size, beam_size, max_iter, attn_num_units,
l2_regularize, word_config, spectrogram_config, lstm_int_num, batch_size,
loss_weight) = get_PEC_config(config)
print("Building model architecture ...")
CE, loss, cla_loss, train_outs, infer_outputs, score = compute_loss(
source_ids, target_ids, sequence_mask, choice_qs, embeddings,
enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,
dec_num_layers, dec_num_units, dec_cell_type, state_pass, num_emo,
emo_cat, emo_cat_units, emo_int_units, infer_batch_size, spectrogram,
word_config, per_num, person_ids, per_embed_size, spectrogram_config,
loss_weight, lstm_int_num, is_train, False, lexicons_ids, beam_size,
max_iter, attn_num_units, l2_regularize, name)
print("\tDone.")
# Even if we restored the model, we will treat it as new training
# if the trained model is written into an arbitrary location.
(logdir, restore_from, learning_rate, gpu_fraction, max_checkpoints,
train_steps, batch_size, print_every, checkpoint_every, s_filename,
t_filename, q_filename, s_max_leng, t_max_leng, dev_s_filename,
dev_t_filename, dev_q_filename, loss_fig, perp_fig, sp_filename,
sp_max_leng, test_s_filename, test_t_filename, test_q_filename,
test_output) = get_training_config(config, "training")
# Set up session
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options))
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
var_list = tf.trainable_variables()
g_list = tf.global_variables()
bn_moving_vars = [g for g in g_list if 'moving_mean' in g.name]
bn_moving_vars += [g for g in g_list if 'moving_variance' in g.name]
var_list += bn_moving_vars
saver = tf.train.Saver(var_list=tf.trainable_variables(),
max_to_keep=max_checkpoints)
try:
saved_global_step = load(saver, sess, logdir)
if saved_global_step is None:
raise ValueError("Cannot find the checkpoint to restore from.")
except Exception:
print("Something went wrong while restoring checkpoint. ")
raise
# ##### Inference #####
# Load data
print("Loading inference data ...")
# id_0, id_1, id_2 preserved for SOS, EOS, constant zero padding
embed_shift = 3
lexicons = load_lexicons() + embed_shift
test_source_sentences_ids, test_source_person, test_source_data = loadfile(
test_s_filename, is_dialog=True, is_source=True, max_length=s_max_leng)
test_source_data += embed_shift
test_target_sentences_ids, test_target_person, test_target_data, test_category_data = loadfile(
test_t_filename,
is_dialog=True,
is_source=False,
max_length=t_max_leng)
test_target_data += embed_shift
test_spectrogram_data = load_spectrogram(sp_filename,
test_source_sentences_ids)
test_choice_data = loadfile(test_q_filename,
is_dialog=False,
is_source=False,
max_length=t_max_leng)
test_choice_data[test_choice_data < 0] = 0
test_choice_data = test_choice_data.astype(np.float32)
test_masks = (test_target_data >= embed_shift)
test_masks = np.append(np.ones([len(test_masks), 1], dtype=bool),
test_masks,
axis=1)
test_masks = test_masks[:, :-1]
print("\tDone.")
# test
print("testing")
if test_source_data is not None:
CE_words = N_words = 0.0
for start in range(0, len(test_source_data), batch_size):
test_feed_dict = {
source_ids: test_source_data[start:start + batch_size],
target_ids: test_target_data[start:start + batch_size],
person_ids: test_target_person[start:start + batch_size],
spectrogram: test_spectrogram_data[start:start + batch_size],
choice_qs: test_choice_data[start:start + batch_size],
emo_cat: test_category_data[start:start + batch_size],
sequence_mask: test_masks[start:start + batch_size],
lexicons_ids: lexicons,
is_train: False,
}
CE_word, N_word = compute_test_perplexity(
sess, CE, test_masks[start:start + batch_size], test_feed_dict)
CE_words += CE_word
N_words += N_word
print("test_perp: {:.3f}".format(np.exp(CE_words / N_words)))
infer_results = []
for start in range(0, len(test_source_data), infer_batch_size):
# infer_result = sess.run(infer_outputs,
# feed_dict={source_ids: test_source_data[start:start + infer_batch_size],
# spectrogram: test_spectrogram_data[start:start + infer_batch_size],
# person_ids: test_target_person[start:start + infer_batch_size],
# emo_cat: test_category_data[start:start + infer_batch_size],
# lexicons_ids: lexicons,
# is_train: False,
# })
#
# infer_result = infer_result.ids[:, :, 0]
# if infer_result.shape[1] < max_iter:
# l_pad = max_iter - infer_result.shape[1]
# infer_result = np.concatenate((infer_result, np.ones((infer_batch_size, l_pad))), axis=1)
# else:
# infer_result = infer_result[:, :max_iter]
tmp_result = []
scores = []
for i in range(num_emo):
cat = i * np.ones(
[len(test_target_person[start:start + infer_batch_size])])
infer_result, sco = sess.run(
[infer_outputs, score],
feed_dict={
source_ids:
test_source_data[start:start + infer_batch_size],
spectrogram:
test_spectrogram_data[start:start + infer_batch_size],
#spectrogram: np.zeros([len(test_source_data[start:start + infer_batch_size]), 400, 200]),
person_ids:
test_target_person[start:start + infer_batch_size],
emo_cat:
cat,
lexicons_ids:
lexicons,
is_train:
False,
})
infer_result = infer_result.ids[:, :, 0]
if infer_result.shape[1] < max_iter:
l_pad = max_iter - infer_result.shape[1]
infer_result = np.concatenate(
(infer_result, np.ones((infer_batch_size, l_pad))),
axis=1)
else:
infer_result = infer_result[:, :max_iter]
tmp_result.append(infer_result)
scores.append(sco)
tmp_result = np.transpose(np.array(tmp_result), [1, 0, 2])
scores = np.array(scores)
scores = np.exp(scores) / np.sum(np.exp(scores), axis=0)
scores = np.transpose(np.array(scores), [1, 0])
scores[range(infer_batch_size),
test_category_data[start:start + infer_batch_size]] += 1
ind = np.argmax(scores, axis=-1)
infer_results.extend(
tmp_result[range(tmp_result.shape[0]),
test_category_data[start:start + infer_batch_size]])
#infer_results.extend(infer_result)
final_result = np.array(infer_results) - embed_shift
final_result[final_result >= vocab_size] -= (vocab_size + embed_shift)
final_result = id2_word(final_result.astype(int).tolist())
with open(os.path.join(test_output, "PEC_out_emo.tsv"), "w") as f:
f.writelines('\n'.join([
"0\t0\t" + str(emo) + "\t" + ' '.join(sen)
for emo, sen in zip(test_category_data, final_result)
]) + '\n')
with open(os.path.join(test_output, "PEC_out_per.tsv"), "w") as f:
f.writelines('\n'.join([
"0\t0\t" + str(per) + "\t" + ' '.join(sen)
for per, sen in zip(test_target_person, final_result)
]) + '\n')
def main(args):
# loading configurations
with open(args.config) as f:
config = yaml.safe_load(f)["configuration"]
name = config["Name"]
# Construct or load embeddings
print("Initializing embeddings ...")
vocab_size = config["embeddings"]["vocab_size"]
embed_size = config["embeddings"]["embed_size"]
per_num = config["embeddings"]["person_num"]
per_embed_size = config["embeddings"]["person_embed_size"]
ori_emb, ori_p_emb = load_embedding("model/emb.tsv")
embeddings = init_embeddings(vocab_size,
embed_size,
initial_values=ori_emb,
name=name)
print("\tDone.")
# Build the model and compute losses
source_ids = tf.placeholder(tf.int32, [None, 40], name="source")
target_ids = tf.placeholder(tf.int32, [None, 40], name="target")
person_ids = tf.placeholder(tf.int32, [None], name="person_ids")
lexicons_ids = tf.placeholder(tf.int32, [per_num, 1000],
name="lexicons_ids")
spectrogram = tf.placeholder(tf.float32, [None, 400, 200], name="audio")
sequence_mask = tf.placeholder(tf.bool, [None, 40], name="mask")
choice_qs = tf.placeholder(tf.float32, [None, 40], name="choice")
emo_cat = tf.placeholder(tf.int32, [None], name="emotion_category")
is_train = tf.placeholder(tf.bool)
(enc_num_layers, enc_num_units, enc_cell_type, enc_bidir, dec_num_layers,
dec_num_units, dec_cell_type, state_pass, num_emo, emo_cat_units,
emo_int_units, infer_batch_size, beam_size, max_iter, attn_num_units,
l2_regularize, word_config, spectrogram_config, lstm_int_num, batch_size,
loss_weight) = get_PEC_config(config)
print("Building model architecture ...")
CE, loss, cla_loss, train_outs, infer_outputs, score = compute_loss(
source_ids, target_ids, sequence_mask, choice_qs, embeddings,
enc_num_layers, enc_num_units, enc_cell_type, enc_bidir,
dec_num_layers, dec_num_units, dec_cell_type, state_pass, num_emo,
emo_cat, emo_cat_units, emo_int_units, infer_batch_size, spectrogram,
word_config, per_num, person_ids, per_embed_size, spectrogram_config,
loss_weight, lstm_int_num, is_train, False, lexicons_ids, beam_size,
max_iter, attn_num_units, l2_regularize, name)
print("\tDone.")
# Even if we restored the model, we will treat it as new training
# if the trained model is written into an arbitrary location.
(logdir, restore_from, learning_rate, gpu_fraction, max_checkpoints,
train_steps, batch_size, print_every, checkpoint_every, s_filename,
t_filename, q_filename, s_max_leng, t_max_leng, dev_s_filename,
dev_t_filename, dev_q_filename, loss_fig, perp_fig, sp_filename,
sp_max_leng, test_s_filename, test_t_filename, test_q_filename,
test_output) = get_training_config(config, "training")
is_overwritten_training = logdir != restore_from
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=1e-4)
trainable = tf.trainable_variables()
gradients = tf.gradients(loss, trainable)
clipped_gradients, gradient_norm = tf.clip_by_global_norm(gradients, 5.0)
optim = optimizer.apply_gradients(zip(clipped_gradients, trainable))
# optim = optimizer.minimize(loss, var_list=trainable)
# Set up session
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options))
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
var_list = tf.trainable_variables()
g_list = tf.global_variables()
bn_moving_vars = [g for g in g_list if 'moving_mean' in g.name]
bn_moving_vars += [g for g in g_list if 'moving_variance' in g.name]
var_list += bn_moving_vars
saver = tf.train.Saver(var_list=tf.trainable_variables(),
max_to_keep=max_checkpoints)
# BN
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
try:
saved_global_step = load(saver, sess, restore_from)
if is_overwritten_training or saved_global_step is None:
# The first training step will be saved_global_step + 1,
# therefore we put -1 here for new or overwritten trainings.
saved_global_step = -1
except Exception:
print("Something went wrong while restoring checkpoint. "
"Training is terminated to avoid the overwriting.")
raise
# ##### Training #####
# Load data
print("Loading data ...")
# id_0, id_1, id_2 preserved for SOS, EOS, constant zero padding
embed_shift = 3
lexicons = load_lexicons() + embed_shift
source_sentences_ids, source_person, source_data = loadfile(
s_filename, is_dialog=True, is_source=True, max_length=s_max_leng)
source_data += embed_shift
target_sentences_ids, target_person, target_data, category_data = loadfile(
t_filename, is_dialog=True, is_source=False, max_length=t_max_leng)
target_data += embed_shift
spectrogram_data = load_spectrogram(sp_filename, source_sentences_ids)
choice_data = loadfile(q_filename,
is_dialog=False,
is_source=False,
max_length=t_max_leng)
choice_data = choice_data.astype(np.float32)
masks = (target_data >= embed_shift)
masks = np.append(np.ones([len(masks), 1], dtype=bool), masks, axis=1)
masks = masks[:, :-1]
n_data = len(source_data)
dev_source_data = None
if dev_s_filename is not None:
dev_source_sentences_ids, dev_source_person, dev_source_data = loadfile(
dev_s_filename,
is_dialog=True,
is_source=True,
max_length=s_max_leng)
dev_source_data += embed_shift
dev_target_sentences_ids, dev_target_person, dev_target_data, dev_category_data = loadfile(
dev_t_filename,
is_dialog=True,
is_source=False,
max_length=t_max_leng)
dev_target_data += embed_shift
dev_spectrogram_data = load_spectrogram(sp_filename,
dev_source_sentences_ids)
dev_choice_data = loadfile(dev_q_filename,
is_dialog=False,
is_source=False,
max_length=t_max_leng)
dev_choice_data[dev_choice_data < 0] = 0
dev_choice_data = dev_choice_data.astype(np.float32)
dev_masks = (dev_target_data >= embed_shift)
dev_masks = np.append(np.ones([len(dev_masks), 1], dtype=bool),
dev_masks,
axis=1)
dev_masks = dev_masks[:, :-1]
print("\tDone.")
# Training
last_saved_step = saved_global_step
num_steps = saved_global_step + train_steps
losses = []
cla_losses = []
steps = []
perps = []
dev_perps = []
print("Start training ...")
try:
step = last_saved_step
for step in range(saved_global_step + 1, num_steps):
start_time = time.time()
rand_indexes = np.random.choice(n_data, batch_size)
source_batch = source_data[rand_indexes]
target_batch = target_data[rand_indexes]
person_batch = target_person[rand_indexes]
spectrogram_batch = spectrogram_data[rand_indexes]
mask_batch = masks[rand_indexes]
choice_batch = choice_data[rand_indexes]
emotions = category_data[rand_indexes]
feed_dict = {
source_ids: source_batch,
target_ids: target_batch,
person_ids: person_batch,
spectrogram: spectrogram_batch,
sequence_mask: mask_batch,
choice_qs: choice_batch,
emo_cat: emotions,
lexicons_ids: lexicons,
is_train: True,
}
loss_value, cla_value, _, __ = sess.run(
[loss, cla_loss, optim, extra_update_ops], feed_dict=feed_dict)
losses.append(loss_value)
cla_losses.append(cla_value)
duration = time.time() - start_time
if step % print_every == 0:
# train perplexity
t_perp = compute_perplexity(sess, CE, mask_batch, feed_dict)
perps.append(t_perp)
# dev perplexity
dev_str = ""
if dev_source_data is not None:
CE_words = N_words = 0.0
for start in range(0, len(dev_source_data), batch_size):
dev_feed_dict = {
source_ids:
dev_source_data[start:start + batch_size],
target_ids:
dev_target_data[start:start + batch_size],
person_ids:
dev_target_person[start:start + batch_size],
spectrogram:
dev_spectrogram_data[start:start + batch_size],
choice_qs:
dev_choice_data[start:start + batch_size],
emo_cat:
dev_category_data[start:start + batch_size],
sequence_mask:
dev_masks[start:start + batch_size],
lexicons_ids:
lexicons,
is_train:
False,
}
CE_word, N_word = compute_test_perplexity(
sess, CE, dev_masks[start:start + batch_size],
dev_feed_dict)
CE_words += CE_word
N_words += N_word
dev_str = "dev_prep: {:.3f}, ".format(
np.exp(CE_words / N_words))
dev_perps.append(np.exp(CE_words / N_words))
steps.append(step)
info = 'step {:d}, loss = {:.6f}, cla_loss = {:.6f} '
info += 'perp: {:.3f}, {}({:.3f} sec/step)'
print(
info.format(step, loss_value, cla_value, t_perp, dev_str,
duration))
if step % checkpoint_every == 0:
save(saver, sess, logdir, step)
last_saved_step = step
except KeyboardInterrupt:
# Introduce a line break after ^C so save message is on its own line.
print()
finally:
if step > last_saved_step:
save(saver, sess, logdir, step)