parser.add_argument('--data-path', dest='data_path', required=True)
parser.add_argument('--train-list', dest='train_list', required=True)
parser.add_argument('--val-list', dest='val_list', default=None)
parser.add_argument('--px-net', dest='px_net_path', required=True)
parser.add_argument('--qy-net', dest='qy_net_path', required=True)
parser.add_argument('--qz-net', dest='qz_net_path', required=True)
parser.add_argument('--pt-net', dest='pt_net_path', required=True)
parser.add_argument('--init-path', dest='init_path', default=None)
parser.add_argument('--preproc-file', dest='preproc_file', default=None)
parser.add_argument('--output-path', dest='output_path', required=True)
VAE.add_argparse_args(parser)
G.add_argparse_args(parser)
KOF.add_argparse_args(parser)
KCF.add_argparse_args(parser)
parser.add_argument('--freeze-embed',
dest='freeze_embed',
default=False,
action='store_true')
parser.add_argument('--epochs', dest='epochs', default=1000, type=int)
parser.add_argument('-v',
'--verbose',
dest='verbose',
default=1,
choices=[0, 1, 2, 3],
type=int)
args = parser.parse_args()
def train_embed(data_path, train_list, val_list, px_net_path, pt_net_path,
qy_net_path, qz_net_path, init_path, epochs, preproc_file,
output_path, freeze_embed, **kwargs):
g = reserve_gpu()
set_float_cpu(float_keras())
if preproc_file is not None:
preproc = TransformList.load(preproc_file)
else:
preproc = None
sg_args = G.filter_args(**kwargs)
sg = G(data_path,
train_list,
shuffle_seqs=True,
reset_rng=False,
transform=preproc,
**sg_args)
max_length = sg.max_seq_length
gen_val = None
if val_list is not None:
sg_val = G(data_path,
val_list,
transform=preproc,
shuffle_seqs=False,
reset_rng=True,
**sg_args)
max_length = max(max_length, sg_val.max_seq_length)
gen_val = data_generator(sg_val, max_length)
gen_train = data_generator(sg, max_length)
if init_path is None:
model, init_epoch = KML.load_checkpoint(output_path, epochs)
if model is None:
embed_args = VAE.filter_args(**kwargs)
logging.debug(embed_args)
px_net = load_model_arch(px_net_path)
qy_net = load_model_arch(qy_net_path)
qz_net = load_model_arch(qz_net_path)
pt_net = load_model_arch(pt_net_path)
model = VAE(px_net, qy_net, qz_net, pt_net, **embed_args)
else:
sg.cur_epoch = init_epoch
sg.reset()
else:
logging.info('loading init model: %s' % init_path)
model = KML.load(init_path)
model.px_weight = kwargs['px_weight']
model.pt_weight = kwargs['pt_weight']
model.kl_qy_weight = kwargs['kl_qy_weight']
model.kl_qz_weight = kwargs['kl_qz_weight']
opt_args = KOF.filter_args(**kwargs)
cb_args = KCF.filter_args(**kwargs)
logging.debug(sg_args)
logging.debug(opt_args)
logging.debug(cb_args)
logging.info('max length: %d' % max_length)
t1 = time.time()
if freeze_embed:
model.prepool_net.trainable = False
model.build(max_length)
logging.info(time.time() - t1)
cb = KCF.create_callbacks(model, output_path, **cb_args)
opt = KOF.create_optimizer(**opt_args)
model.compile(optimizer=opt)
h = model.fit_generator(gen_train,
validation_data=gen_val,
steps_per_epoch=sg.steps_per_epoch,
validation_steps=sg_val.steps_per_epoch,
initial_epoch=sg.cur_epoch,
epochs=epochs,
callbacks=cb,
max_queue_size=10)
logging.info('Train elapsed time: %.2f' % (time.time() - t1))
model.save(output_path + '/model')
def train_embed(seq_file, train_list, val_list, class_list, embed_file,
init_path, epochs, batch_size, preproc_file, output_path,
post_pdf, pooling_input, pooling_output, min_var, **kwargs):
set_float_cpu(float_keras())
sr_args = SR.filter_args(**kwargs)
sr_val_args = SR.filter_val_args(**kwargs)
opt_args = KOF.filter_args(**kwargs)
cb_args = KCF.filter_args(**kwargs)
if preproc_file is not None:
preproc = TransformList.load(preproc_file)
else:
preproc = None
sr = SR(seq_file,
train_list,
class_list,
batch_size=batch_size,
preproc=preproc,
**sr_args)
max_length = sr.max_batch_seq_length
gen_val = None
if val_list is not None:
sr_val = SR(seq_file,
val_list,
class_list,
batch_size=batch_size,
preproc=preproc,
shuffle_seqs=False,
seq_split_mode='sequential',
seq_split_overlap=0,
reset_rng=True,
**sr_val_args)
max_length = max(max_length, sr_val.max_batch_seq_length)
gen_val = data_generator(sr_val, max_length)
gen_train = data_generator(sr, max_length)
t1 = time.time()
if init_path is None:
embed_net = load_model_arch(embed_file)
model = SeqMetaEmbed(embed_net,
num_classes=sr.num_classes,
post_pdf=post_pdf,
pooling_input=pooling_input,
pooling_output=pooling_output,
min_var=min_var)
else:
logging.info('loading init model: %s' % init_path)
model = SeqMetaEmbed.load(init_path)
logging.info('max length: %d' % max_length)
model.build(max_length)
logging.info(time.time() - t1)
cb = KCF.create_callbacks(model, output_path, **cb_args)
opt = KOF.create_optimizer(**opt_args)
model.compile(optimizer=opt)
h = model.fit_generator(gen_train,
validation_data=gen_val,
steps_per_epoch=sr.num_batches,
validation_steps=sr_val.num_batches,
epochs=epochs,
callbacks=cb,
max_queue_size=10)
logging.info('Train elapsed time: %.2f' % (time.time() - t1))
model.save(output_path + '/model')
def train_embed(data_path, train_list, val_list,
train_list_adapt, val_list_adapt,
prepool_net_path, postpool_net_path,
init_path,
epochs,
preproc_file, output_path,
freeze_prepool, freeze_postpool_layers,
**kwargs):
set_float_cpu(float_keras())
if init_path is None:
model, init_epoch = KML.load_checkpoint(output_path, epochs)
if model is None:
emb_args = SeqEmbed.filter_args(**kwargs)
prepool_net = load_model_arch(prepool_net_path)
postpool_net = load_model_arch(postpool_net_path)
model = SeqEmbed(prepool_net, postpool_net,
loss='categorical_crossentropy',
**emb_args)
else:
kwargs['init_epoch'] = init_epoch
else:
logging.info('loading init model: %s' % init_path)
model = KML.load(init_path)
sg_args = G.filter_args(**kwargs)
opt_args = KOF.filter_args(**kwargs)
cb_args = KCF.filter_args(**kwargs)
logging.debug(sg_args)
logging.debug(opt_args)
logging.debug(cb_args)
if preproc_file is not None:
preproc = TransformList.load(preproc_file)
else:
preproc = None
sg = G(data_path, train_list, train_list_adapt,
shuffle_seqs=True, reset_rng=False,
transform=preproc, **sg_args)
max_length = sg.max_seq_length
gen_val = None
if val_list is not None:
sg_val = G(data_path, val_list, val_list_adapt,
transform=preproc,
shuffle_seqs=False, reset_rng=True,
**sg_args)
max_length = max(max_length, sg_val.max_seq_length)
gen_val = data_generator(sg_val, max_length)
gen_train = data_generator(sg, max_length)
logging.info('max length: %d' % max_length)
t1 = time.time()
if freeze_prepool:
model.freeze_prepool_net()
if freeze_postpool_layers is not None:
model.freeze_postpool_net_layers(freeze_postpool_layers)
model.build(max_length)
cb = KCF.create_callbacks(model, output_path, **cb_args)
opt = KOF.create_optimizer(**opt_args)
model.compile(optimizer=opt)
h = model.fit_generator(gen_train, validation_data=gen_val,
steps_per_epoch=sg.steps_per_epoch,
validation_steps=sg_val.steps_per_epoch,
initial_epoch=sg.cur_epoch,
epochs=epochs, callbacks=cb, max_queue_size=10)
logging.info('Train elapsed time: %.2f' % (time.time() - t1))
model.save(output_path + '/model')
def train_tvae(seq_file, train_list, val_list,
decoder_file, qy_file, qz_file,
epochs, batch_size,
preproc_file, output_path,
num_samples_y, num_samples_z,
px_form, qy_form, qz_form,
min_kl, **kwargs):
set_float_cpu(float_keras())
sr_args = SR.filter_args(**kwargs)
sr_val_args = SR.filter_val_args(**kwargs)
opt_args = KOF.filter_args(**kwargs)
cb_args = KCF.filter_args(**kwargs)
if preproc_file is not None:
preproc = TransformList.load(preproc_file)
else:
preproc = None
sr = SR(seq_file, train_list, batch_size=batch_size,
preproc=preproc, **sr_args)
max_length = sr.max_batch_seq_length
gen_val = None
if val_list is not None:
sr_val = SR(seq_file, val_list, batch_size=batch_size,
preproc=preproc,
shuffle_seqs=False,
seq_split_mode='sequential', seq_split_overlap=0,
reset_rng=True,
**sr_val_args)
max_length = max(max_length, sr_val.max_batch_seq_length)
gen_val = data_generator(sr_val, max_length)
gen_train = data_generator(sr, max_length)
t1 = time.time()
decoder = load_model_arch(decoder_file)
qy = load_model_arch(qy_file)
if qz_file is None:
vae = TVAEY(qy, decoder, px_cond_form=px_form,
qy_form=qy_form, min_kl=min_kl)
vae.build(num_samples=num_samples_y,
max_seq_length = max_length)
else:
qz = load_model_arch(qz_file)
vae = TVAEYZ(qy, qz, decoder, px_cond_form=px_form,
qy_form=qy_form, qz_form=qz_form, min_kl=min_kl)
vae.build(num_samples_y=num_samples_y, num_samples_z=num_samples_z,
max_seq_length = max_length)
logging.info(time.time()-t1)
cb = KCF.create_callbacks(vae, output_path, **cb_args)
opt = KOF.create_optimizer(**opt_args)
h = vae.fit_generator(gen_train, x_val=gen_val,
steps_per_epoch=sr.num_batches,
validation_steps=sr_val.num_batches,
optimizer=opt, epochs=epochs,
callbacks=cb, max_q_size=10)
# if vae.x_chol is not None:
# x_chol = np.array(K.eval(vae.x_chol))
# logging.info(x_chol[:4,:4])
logging.info('Train elapsed time: %.2f' % (time.time() - t1))
vae.save(output_path + '/model')
def train_tvae(seq_file, train_list, val_list, class_list, post_file,
decoder_file, pt_file, qy_file, qz_file, init_path, epochs,
batch_size, preproc_file, output_path, num_samples_y,
num_samples_z, px_form, qy_form, qz_form, min_kl, loss_weights,
**kwargs):
set_float_cpu(float_keras())
sr_args = SR.filter_args(**kwargs)
sr_val_args = SR.filter_val_args(**kwargs)
opt_args = KOF.filter_args(**kwargs)
cb_args = KCF.filter_args(**kwargs)
if preproc_file is not None:
preproc = TransformList.load(preproc_file)
else:
preproc = None
sr = SR(seq_file,
train_list,
post_file,
class_list,
batch_size=batch_size,
preproc=preproc,
**sr_args)
max_length = sr.max_batch_seq_length
gen_val = None
if val_list is not None:
sr_val = SR(seq_file,
val_list,
post_file,
class_list,
batch_size=batch_size,
preproc=preproc,
shuffle_seqs=False,
seq_split_mode='sequential',
seq_split_overlap=0,
reset_rng=True,
**sr_val_args)
max_length = max(max_length, sr_val.max_batch_seq_length)
gen_val = data_generator(sr_val, max_length)
gen_train = data_generator(sr, max_length)
t1 = time.time()
decoder = load_model_arch(decoder_file)
qy = load_model_arch(qy_file)
pt = load_model_arch(pt_file)
if init_path is None:
decoder = load_model_arch(decoder_file)
qy = load_model_arch(qy_file)
# if qz_file is None:
# vae = TVAEY(qy, decoder, px_cond_form=px_form,
# qy_form=qy_form, min_kl=min_kl)
# vae.build(num_samples=num_samples_y,
# max_seq_length = max_length)
# else:
qz = load_model_arch(qz_file)
vae = TVAEYZ(qy,
qz,
decoder,
pt,
px_cond_form=px_form,
qy_form=qy_form,
qz_form=qz_form,
min_kl=min_kl,
loss_weights=loss_weights)
else:
logging.info('loading init model: %s' % init_path)
vae = TVAEYZ.load(init_path)
logging.info('max length: %d' % max_length)
vae.build(num_samples_y=num_samples_y,
num_samples_z=num_samples_z,
max_seq_length=max_length)
logging.info(time.time() - t1)
cb = KCF.create_callbacks(vae, output_path, **cb_args)
opt = KOF.create_optimizer(**opt_args)
h = vae.fit_generator(gen_train,
x_val=gen_val,
steps_per_epoch=sr.num_batches,
validation_steps=sr_val.num_batches,
optimizer=opt,
epochs=epochs,
callbacks=cb,
max_queue_size=10)
# if vae.x_chol is not None:
# x_chol = np.array(K.eval(vae.x_chol))
# logging.info(x_chol[:4,:4])
logging.info('Train elapsed time: %.2f' % (time.time() - t1))
vae.save(output_path + '/model')
sr_val.reset()
y_val, sy_val, z_val, srz_val = vae.encoder_net.predict_generator(
gen_val, steps=400)
from scipy import linalg as la
yy = y_val - np.mean(y_val, axis=0)
cy = np.dot(yy.T, yy) / yy.shape[0]
l, v = la.eigh(cy)
np.savetxt(output_path + '/l1.txt', l)
sr_val.reset()
y_val2, sy_val2 = vae.qy_net.predict_generator(gen_val, steps=400)
yy = y_val2 - np.mean(y_val, axis=0)
cy = np.dot(yy.T, yy) / yy.shape[0]
l, v = la.eigh(cy)
np.savetxt(output_path + '/l2.txt', l)
logging.info(y_val - y_val2)
def train_pdda(iv_file, train_list, val_list,
decoder_file, qy_file, qz_file,
epochs, batch_size,
preproc_file, output_path,
num_samples_y, num_samples_z,
px_form, qy_form, qz_form,
min_kl, **kwargs):
set_float_cpu('float32')
vcr_args = VCR.filter_args(**kwargs)
opt_args = KOF.filter_args(**kwargs)
cb_args = KCF.filter_args(**kwargs)
if preproc_file is not None:
preproc = TransformList.load(preproc_file)
else:
preproc = None
vcr_train = VCR(iv_file, train_list, preproc, **vcr_args)
max_length = vcr_train.max_samples_per_class
x_val = None
sw_val = None
if val_list is not None:
vcr_val = VCR(iv_file, val_list, preproc, **vcr_args)
max_length = max(max_length, vcr_val.max_samples_per_class)
x_val, sw_val = vcr_val.read(return_3d=True, max_length=max_length)
x, sw = vcr_train.read(return_3d=True, max_length=max_length)
t1 = time.time()
decoder = load_model_arch(decoder_file)
qy = load_model_arch(qy_file)
if qz_file is None:
vae = TVAEY(qy, decoder, px_cond_form=px_form,
qy_form=qy_form, min_kl=min_kl)
vae.build(num_samples=num_samples_y,
max_seq_length = x.shape[1])
else:
qz = load_model_arch(qz_file)
vae = TVAEYZ(qy, qz, decoder, px_cond_form=px_form,
qy_form=qy_form, qz_form=qz_form, min_kl=min_kl)
vae.build(num_samples_y=num_samples_y, num_samples_z=num_samples_z,
max_seq_length = x.shape[1])
logging.info(time.time()-t1)
# opt = create_optimizer(**opt_args)
# cb = create_basic_callbacks(vae, output_path, **cb_args)
# h = vae.fit(x, x_val=x_val,
# sample_weight_train=sw, sample_weight_val=sw_val,
# optimizer=opt, shuffle=True, epochs=100,
# batch_size=batch_size, callbacks=cb)
# opt = create_optimizer(**opt_args)
# cb = create_basic_callbacks(vae, output_path, **cb_args)
# h = vae.fit_mdy(x, x_val=x_val,
# sample_weight_train=sw, sample_weight_val=sw_val,
# optimizer=opt, shuffle=True, epochs=200,
# batch_size=batch_size, callbacks=cb)
# y_mean, y_logvar, z_mean, z_logvar = vae.compute_qyz_x(
# x, batch_size=batch_size)
# sw = np.expand_dims(sw, axis=-1)
# m_y = np.mean(np.mean(y_mean, axis=0))
# s2_y = np.sum(np.sum(np.exp(y_logvar)+y_mean**2, axis=0)/
# y_logvar.shape[0]-m_y**2)
# m_z = np.mean(np.sum(np.sum(z_mean*sw, axis=1), axis=0)
# /np.sum(sw))
# s2_z = np.sum(np.sum(np.sum((np.exp(z_logvar)+z_mean**2)*sw, axis=1), axis=0)
# /np.sum(sw)-m_z**2)
# logging.info('m_y: %.2f, trace_y: %.2f, m_z: %.2f, trace_z: %.2f' %
# (m_y, s2_y, m_z, s2_z))
cb = KCF.create_callbacks(vae, output_path, **cb_args)
opt = KOF.create_optimizer(**opt_args)
h = vae.fit(x, x_val=x_val,
sample_weight_train=sw, sample_weight_val=sw_val,
optimizer=opt, shuffle=True, epochs=epochs,
batch_size=batch_size, callbacks=cb)
if vae.x_chol is not None:
x_chol = np.array(K.eval(vae.x_chol))
logging.info(x_chol[:4,:4])
logging.info('Train elapsed time: %.2f' % (time.time() - t1))
vae.save(output_path + '/model')
t1 = time.time()
elbo = np.mean(vae.elbo(x, num_samples=1, batch_size=batch_size))
logging.info('elbo: %.2f' % elbo)
logging.info('Elbo elapsed time: %.2f' % (time.time() - t1))
t1 = time.time()
vae.build(num_samples_y=1, num_samples_z=1, max_seq_length = x.shape[1])
vae.compile()
qyz = vae.compute_qyz_x(x, batch_size=batch_size)
if vae.qy_form == 'diag_normal':
y_mean, y_logvar = qyz[:2]
qz = qyz[2:]
else:
y_mean, y_logvar, y_chol = qyz[:3]
qz = qyz[3:]
if vae.qz_form == 'diag_normal':
z_mean, z_logvar = qz[:2]
else:
z_mean, z_logvar, z_chol = qz[:3]
sw = np.expand_dims(sw, axis=-1)
m_y = np.mean(np.mean(y_mean, axis=0))
s2_y = np.sum(np.sum(np.exp(y_logvar)+y_mean**2, axis=0)/
y_logvar.shape[0]-m_y**2)
m_z = np.mean(np.sum(np.sum(z_mean*sw, axis=1), axis=0)
/np.sum(sw))
s2_z = np.sum(np.sum(np.sum((np.exp(z_logvar)+z_mean**2)*sw, axis=1), axis=0)
/np.sum(sw)-m_z**2)
logging.info('m_y: %.2f, trace_y: %.2f, m_z: %.2f, trace_z: %.2f' %
(m_y, s2_y, m_z, s2_z))
logging.info('Trace elapsed time: %.2f' % (time.time() - t1))
t1 = time.time()
vae.build(num_samples_y=1, num_samples_z=1, max_seq_length = 2)
vae.compile()
x1 = x[:,0,:]
x2 = x[:,1,:]
# scores = vae.eval_llr_1vs1_elbo(x1, x2, num_samples=10)
# tar = scores[np.eye(scores.shape[0], dtype=bool)]
# non = scores[np.logical_not(np.eye(scores.shape[0], dtype=bool))]
# logging.info('m_tar: %.2f s_tar: %.2f' % (np.mean(tar), np.std(tar)))
# logging.info('m_non: %.2f s_non: %.2f' % (np.mean(non), np.std(non)))
# scores = vae.eval_llr_1vs1_cand(x1, x2)
# tar = scores[np.eye(scores.shape[0], dtype=bool)]
# non = scores[np.logical_not(np.eye(scores.shape[0], dtype=bool))]
# logging.info('m_tar: %.2f s_tar: %.2f' % (np.mean(tar), np.std(tar)))
# logging.info('m_non: %.2f s_non: %.2f' % (np.mean(non), np.std(non)))
scores = vae.eval_llr_1vs1_qscr(x1, x2)
tar = scores[np.eye(scores.shape[0], dtype=bool)]
non = scores[np.logical_not(np.eye(scores.shape[0], dtype=bool))]
logging.info('m_tar: %.2f s_tar: %.2f' % (np.mean(tar), np.std(tar)))
logging.info('m_non: %.2f s_non: %.2f' % (np.mean(non), np.std(non)))
logging.info('Eval elapsed time: %.2f' % (time.time() - t1))