def __init__(self, params, datas, model_dict):
super(BVA, self).__init__()
# embedding layer
self.embeddings = XlingEmbeddingLayer(params, datas)
# encoder
self.encoder = Encoder(params)
# discriminator for adversarial training
self.adv_training = params.adv_training
if self.adv_training:
self.discriminator = XlingDiscriminator(params)
# inferer
self.inferer = Inferer(params, params.inf_in_dim)
# vae type
self.vae_type = params.vae_type
# decoder
# untie the embeddings
if params.tie_emb:
self.decoder = XlingDecoder(params, datas, self.embeddings)
else:
embeddings = copy.deepcopy(self.embeddings)
"""
# random initialization
for emb in embeddings.embeddings:
initrange = 0.5 / 300
emb.embeddings.weight.data.uniform_(-initrange, initrange)
emb.embeddings.weight.data[0] = 0
"""
self.decoder = XlingDecoder(params, datas, embeddings)
# load pretrained model
if model_dict is not None:
self.init_model(self, model_dict)
# load pretrained embeddings again in case we need
if model_dict is not None and params.pretrained_emb_path[0] is not None:
for i, lang in enumerate(self.embeddings.lang_dict):
if params.pretrained_emb_path[i] is not None:
vocab = datas[i].vocab
assert (lang == vocab.lang)
self.embeddings.embeddings[i].init_emb(
self.embeddings.embeddings[i].embeddings,
params.pretrained_emb_path[i],
vocab)
self.use_cuda = params.cuda
if self.use_cuda:
self.cuda()
def __init__(self, params, data_x, data_y):
super(BVA, self).__init__()
# encoder
self.encoder_x = Encoder(params, data_x.vocab)
self.encoder_y = Encoder(params, data_y.vocab)
# inferer
self.inferer = Inferer(params)
# number of samples from z
self.sample_n = params.sample_n
self.ls_type = params.ls_type
# decoder
self.decoder = Decoder(params, data_x, data_y)
self.use_cuda = params.cuda
if self.use_cuda:
self.cuda()
def __init__(self, params, data_list, classifier_config, model_dict=None):
super(CLDCModel, self).__init__()
# embedding layer
self.embeddings = XlingEmbeddingLayer(params, data_list)
# encoder
self.encoder = Encoder(params)
# inferer
self.inferer = Inferer(params, params.inf_in_dim)
# load pretrained model
if model_dict is not None:
XlingVA.init_model(self, model_dict)
# CLDC classifier
self.cldc_classifier = CLDCClassifier(params, classifier_config)
self.use_cuda = params.cuda
if self.use_cuda:
self.cuda()
def run():
"""
Prepares and runs the whole system.
"""
args = parse_args()
logger.info('Args are: {}'.format(args))
env = Env(args)
model = env.model
datasets = env.datasets
worker = Trainer(args, model=model, datasets=datasets) \
if not args.is_infer else \
Inferer(args, model=model, datasets=datasets)
worker.start()
class CLDCModel(nn.Module):
def __init__(self, params, data_list, classifier_config, model_dict=None):
super(CLDCModel, self).__init__()
# embedding layer
self.embeddings = XlingEmbeddingLayer(params, data_list)
# encoder
self.encoder = Encoder(params)
# inferer
self.inferer = Inferer(params, params.inf_in_dim)
# load pretrained model
if model_dict is not None:
XlingVA.init_model(self, model_dict)
# CLDC classifier
self.cldc_classifier = CLDCClassifier(params, classifier_config)
self.use_cuda = params.cuda
if self.use_cuda:
self.cuda()
def get_gaus(self, lang, batch_in, batch_lens):
# embedding
input_word_embs = self.embeddings(lang, batch_in)
# encoding
hid = self.encoder(input_word_embs, batch_lens)
# infering
mu, logvar = self.inferer(hid)
return mu, logvar
def forward(self, lang, batch_in, batch_lens, batch_lb=None, vis=False):
# embedding
input_word_embs = self.embeddings(lang, batch_in)
# encoding
hid = self.encoder(input_word_embs, batch_lens)
# infering
mu, logvar = self.inferer(hid)
z = self.inferer.reparameterize(mu, logvar)
# classifier
loss, pred_p, pred = self.cldc_classifier(z,
batch_lb,
self.training,
vis=vis)
#loss, pred_p, pred = self.cldc_classifier(hid, batch_lb, self.training, vis = vis)
return loss, pred_p, pred
# import lib
FACE_EXPRESSION_SRC_ROOT = "/Vol0/user/k.iskakov/dev/face_expression"
sys.path.append(FACE_EXPRESSION_SRC_ROOT)
from inferer import Inferer
# setup device and checkpoint
device = 'cuda:0'
config_path = "/Vol1/dbstore/datasets/k.iskakov/share/face_expression/gold_checkpoints/siamese+mediapipe_normalization+use_beta-false+checkpoint_000018/config.yaml"
checkpoint_path = "/Vol1/dbstore/datasets/k.iskakov/share/face_expression/gold_checkpoints/siamese+mediapipe_normalization+use_beta-false+checkpoint_000018/checkpoint_000018.pth"
# device = 'cpu'
device = 'cuda:0'
inferer = Inferer(config_path,
checkpoint_path,
K_rgb,
T_depth_to_rgb,
device=device)
count = 0
times = []
for fid in tqdm(frame_ids):
print()
if fid not in joints_poses_dict:
continue
joints_poses = joints_poses_dict[fid]
# load rgb image
img_path = root_folder_rgb + '/' + pid_lbl + '/' + dev_lbl + '/color_undistorted/' + str(
fid).zfill(6) + '.jpg'
generator = PCLGen((perspective.DST_HEIGHT, perspective.DST_WIDTH), 50)
if __name__ == "__main__":
import signal
import sys
last = False
lastPCL = None
rospy.init_node("freespace_stopper")
pcl_pub = rospy.Publisher("/seg/pcl", PointCloud, queue_size=10)
scan_sub = rospy.Subscriber("/scan", LaserScan, callback1)
forward_pub = rospy.Publisher("/forward", String, queue_size=10)
image_pub = rospy.Publisher("seg/reg", Image, queue_size=2)
bridge = CvBridge()
inferer = Inferer(2502)
def end(sig, frame):
print "\n\nClosing TF sessions"
inferer.sess.close()
print "done."
sys.exit(0)
signal.signal(signal.SIGINT, end)
cam = WebcamVideoStream(src=1).start()
while True:
frame = cam.read()
start = current_time()
frame = perspective.undistort_internal(frame)
frame = frame[240:, :]
frame = cv2.resize(frame, (480, 180))
class BVA(nn.Module):
def __init__(self, params, data_x, data_y):
super(BVA, self).__init__()
# encoder
self.encoder_x = Encoder(params, data_x.vocab)
self.encoder_y = Encoder(params, data_y.vocab)
# inferer
self.inferer = Inferer(params)
# number of samples from z
self.sample_n = params.sample_n
self.ls_type = params.ls_type
# decoder
self.decoder = Decoder(params, data_x, data_y)
self.use_cuda = params.cuda
if self.use_cuda:
self.cuda()
def forward(self, batch_x, batch_x_lens, batch_y, batch_y_lens):
# encoding
hid_x = self.encoder_x(batch_x, batch_x_lens)
hid_y = self.encoder_y(batch_y, batch_y_lens)
# infering
mu, logvar = self.inferer([hid_x, hid_y])
if self.ls_type == 'nlldetz':
# deterministic z
samp_hid = mu.unsqueeze(1).repeat(1, self.sample_n, 1)
samp_batch_x = batch_x.unsqueeze(-1).repeat(1, 1, self.sample_n)
samp_batch_y = batch_y.unsqueeze(-1).repeat(1, 1, self.sample_n)
else:
# sample n times from z
samp_mu = mu.unsqueeze(1).repeat(1, self.sample_n, 1)
samp_logvar = logvar.unsqueeze(1).repeat(1, self.sample_n, 1)
samp_hid = self.inferer.reparameterize(samp_mu, samp_logvar)
samp_batch_x = batch_x.unsqueeze(-1).repeat(1, 1, self.sample_n)
samp_batch_y = batch_y.unsqueeze(-1).repeat(1, 1, self.sample_n)
# decoding, neg log likelihood
nll = self.decoder(samp_hid, samp_batch_x, samp_batch_y)
# kl divergence, 2 times, considering 2 languages
kld = 2 * torch.mean(
-0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp(), dim=1))
return nll, kld
def save_embedding(self, params, data, xory):
"""
Save the model and the embeddings
"""
self.eval()
lang = data.vocab.lang
word2idx = data.vocab.word2idx
idx2word = data.vocab.idx2word
if xory == 'x':
embeddings = self.decoder.embeddings_x
else:
embeddings = self.decoder.embeddings_y
emb_out = '{}.{}.txt'.format(lang, params.emb_out_path)
print('Saving {}'.format(emb_out))
# save embedding
word_idxs = []
words = []
with open(emb_out, 'w') as fout:
#fout.write('{} {}\n'.format(len(idx2word), params.emb_dim))
for word_idx, word in idx2word.items():
word_idxs.append(word_idx)
words.append(word)
if len(word_idxs) < params.bs:
continue
self.dump_emb_str(embeddings, word_idxs, words, fout)
word_idxs = []
words = []
self.dump_emb_str(embeddings, word_idxs, words, fout)
def dump_emb_str(self, embeddings, word_idxs, words, fout):
assert (len(word_idxs) == len(words))
word_idxs = torch.LongTensor(
word_idxs).cuda() if self.use_cuda else torch.LongTensor(word_idxs)
word_embs = embeddings(word_idxs)
word_embs = word_embs.data.cpu().numpy().tolist()
word_embs = list(zip(words, word_embs))
word_embs = [
'{} {}'.format(w[0], ' '.join(list(map(lambda x: str(x), w[1]))))
for w in word_embs
]
fout.write('{}\n'.format('\n'.join(word_embs)))
def save_model(self, params, data_x, data_y, optimizer):
self.eval()
model_out = '{}.pth'.format(params.emb_out_path)
print('Saving {}'.format(model_out))
# save model
model_dict = {
'model': self.state_dict(),
'optimizer': optimizer.state_dict(),
'vocab_x': data_x.vocab,
'vocab_y': data_y.vocab
}
torch.save(model_dict, model_out)
class XlingVA(BVA):
def __init__(self, params, datas, model_dict):
super(BVA, self).__init__()
# embedding layer
self.embeddings = XlingEmbeddingLayer(params, datas)
# encoder
self.encoder = Encoder(params)
# discriminator for adversarial training
self.adv_training = params.adv_training
if self.adv_training:
self.discriminator = XlingDiscriminator(params)
# inferer
self.inferer = Inferer(params, params.inf_in_dim)
# vae type
self.vae_type = params.vae_type
# decoder
# untie the embeddings
if params.tie_emb:
self.decoder = XlingDecoder(params, datas, self.embeddings)
else:
embeddings = copy.deepcopy(self.embeddings)
"""
# random initialization
for emb in embeddings.embeddings:
initrange = 0.5 / 300
emb.embeddings.weight.data.uniform_(-initrange, initrange)
emb.embeddings.weight.data[0] = 0
"""
self.decoder = XlingDecoder(params, datas, embeddings)
# load pretrained model
if model_dict is not None:
self.init_model(self, model_dict)
# load pretrained embeddings again in case we need
if model_dict is not None and params.pretrained_emb_path[0] is not None:
for i, lang in enumerate(self.embeddings.lang_dict):
if params.pretrained_emb_path[i] is not None:
vocab = datas[i].vocab
assert (lang == vocab.lang)
self.embeddings.embeddings[i].init_emb(
self.embeddings.embeddings[i].embeddings,
params.pretrained_emb_path[i],
vocab)
self.use_cuda = params.cuda
if self.use_cuda:
self.cuda()
@staticmethod
def init_model(obj, model_dict):
# get the current module parameter dicts
cur_model_dict = obj.state_dict()
# 1. filter out unnecessary keys
filtered_model_dict = {}
try:
# old code model
for k, v in model_dict.items():
if xlingva_param_map[k] not in cur_model_dict:
continue
filtered_model_dict[xlingva_param_map[k]] = v
except:
# new code model
filtered_model_dict = {k: model_dict[k] for k in model_dict if k in cur_model_dict}
# only for MVAE in CLDC de case
emb_dict = [k for k in filtered_model_dict if 'embedding' in k]
if filtered_model_dict[emb_dict[0]].shape[0] == 50002:
filtered_model_dict['embeddings.embeddings.1.embeddings.weight'] = filtered_model_dict['embeddings.embeddings.0.embeddings.weight']
del filtered_model_dict['embeddings.embeddings.0.embeddings.weight']
# only for MVAE in CLDC de case
#assert(set(filtered_model_dict.keys()) == set(cur_model_dict.keys()))
# 2. overwrite entries in the existing state dict
cur_model_dict.update(filtered_model_dict)
# 3. load the new state dict
obj.load_state_dict(cur_model_dict)
def get_hidx(self, lang, batch_in, batch_lens):
input_word_embs = self.embeddings(lang, batch_in)
# encoding
hid = self.encoder(input_word_embs, batch_lens)
return hid
def get_gaus(self, lang, batch_in, batch_lens):
# get hidden rep of x
hid = self.get_hidx(lang, batch_in, batch_lens)
# infering
mu, logvar = self.inferer(hid)
loss_dis, loss_enc = torch.tensor(.0), torch.tensor(.0)
if self.adv_training:
# discriminator
loss_dis, loss_enc = self.discriminator(lang, hid)
return mu, logvar, hid, loss_dis, loss_enc
def forward(self, lang, batch_in, batch_lens):
# get hidden rep of x
hid = self.get_hidx(lang, batch_in, batch_lens)
loss_dis, loss_enc = None, None
if self.adv_training:
# discriminator
loss_dis, loss_enc = self.discriminator(lang, hid)
# infering
mu, logvar = self.inferer(hid)
if self.vae_type == 'nlldetz':
# deterministic z
samp_hid = mu.unsqueeze(1).repeat(1, 1, 1)
samp_batch_in = batch_in.unsqueeze(-1).repeat(1, 1, 1)
else:
# vae
hid_rec = self.inferer.reparameterize(mu, logvar)
# decoding
# neg log likelihood
nll = self.decoder(lang, hid_rec, batch_in)
# kl divergence
kld = torch.mean(-0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp(), dim = 1))
return nll, kld, loss_dis, loss_enc
def save_embedding(self, params, data):
self.eval()
lang = data.vocab.lang
if params.seg_type == 'word':
word2idx = data.vocab.word2idx
idx2word = data.vocab.idx2word
elif params.seg_type == 'spm':
word2idx = data.vocab.tok2idx
idx2word = data.vocab.idx2tok
embeddings = self.embeddings.get_lang_emb(lang)
emb_out = '{}.{}.txt'.format(lang, params.log_path)
print('Saving {}'.format(emb_out))
# save embedding
word_idxs = []
words = []
with open(emb_out, 'w') as fout:
#fout.write('{} {}\n'.format(len(idx2word), params.emb_dim))
for word_idx, word in idx2word.items():
word_idxs.append(word_idx)
words.append(word)
if len(word_idxs) < params.test_bs:
continue
super(XlingVA, self).dump_emb_str(embeddings, word_idxs, words, fout)
word_idxs = []
words = []
super(XlingVA, self).dump_emb_str(embeddings, word_idxs, words, fout)
def save_model(self, params, datas):
self.eval()
model_out = '{}.pth'.format(params.log_path)
model_dict = {'model': {k: v.cpu() for k, v in self.state_dict().items()}}
# save model
for lang in params.langs:
lang_idx = params.lang_dict[lang]
model_dict[lang] = datas[lang_idx].vocab
torch.save(model_dict, model_out)