def main():
tau0 = 2.
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_reinforce_smoothed"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(0.05)
reg = EntropyRegularizer(1e-5)
initial_b = np.log(np.sum(cooccurrence, axis=0))
initial_pz_weight = uniform_initializer(5.)((cooccurrence.shape[0], z_k))
model = ReinforceSmoothedModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer,
initial_pz_weight=initial_pz_weight,
initial_b=initial_b,
tau0=tau0)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 4096
z_k = 256
initializer = uniform_initializer(0.05)
scale = 0.05
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
# initial_pz_weight = np.random.uniform(low=-scale, high=scale, size=(cooccurrence.shape[0], z_k))
opt = Adam(1e-3)
tao0 = 5.
tao_min = 0.25
tao_decay = 1e-5
pz_regularizer = EntropyRegularizer(3e-6)
# build and train
outputpath = "output/skipgram_256_gumbel1"
initial_b = np.log(np.sum(cooccurrence, axis=0))
model = GumbelModel1(
cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
initial_b=initial_b,
# initial_pz_weight=initial_pz_weight,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 2048
z_k = 256
initializer = uniform_initializer(0.05)
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
tao0 = 5.
tao_min = 0.1
tao_decay = 1e-4
pz_regularizer = EntropyRegularizer(3e-6)
# build and train
outputpath = "output/skipgram_256_gumbel2"
model = GumbelModel2(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
output_path = 'output/lstm_softmax_vanilla'
epochs = 100
batches = 10000
units = 1024
depth = 35
batch_size = 64
zoneout = 0.5
input_droput = 0.5
output_dropout = 0.5
initializer = uniform_initializer(0.05)
opt = Adam(1e-3)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
model = LSTMSoftmaxVanilla(units=units,
vocab=vocab,
opt=opt,
zoneout=zoneout,
srng=srng,
input_droput=input_droput,
output_dropout=output_dropout,
initializer=initializer)
model.train(output_path=output_path,
epochs=epochs,
batches=batches,
depth=depth,
batch_size=batch_size,
xtrain=xtrain,
xvalid=xvalid,
xtest=xtest)
def main():
output_path = 'output/ganlm'
epochs = 100
batches = 1000
val_batches = 1000
d_batches = 5
depth = 12
batch_size = 64
tau0 = 1.
tau_decay = 0
tau_min = 0.2
d_layers = 2
g_layers = 2
d_units = 512
g_units = 1024
d_dropout = 0.5
dopt = Adam(1e-3)
gopt = Adam(1e-5)
regularizer = l2(1e-5)
regularizer_samples = 128
regularizer_weight = 1e1
#constraint = clip_constraint(1e-1)
constraint = None
initializer = uniform_initializer(0.05)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
model = GANLanguageModel(vocab=vocab,
d_units=d_units,
g_units=g_units,
initializer=initializer,
dopt=dopt,
gopt=gopt,
srng=srng,
d_layers=d_layers,
d_dropout=d_dropout,
g_layers=g_layers,
regularizer=regularizer,
regularizer_samples=regularizer_samples,
regularizer_weight=regularizer_weight,
constraint=constraint,
tau0=tau0,
tau_decay=tau_decay,
tau_min=tau_min)
model.train(output_path=output_path,
epochs=epochs,
batches=batches,
depth=depth,
d_batches=d_batches,
batch_size=batch_size,
val_batches=val_batches,
xtrain=xtrain,
xvalid=xvalid,
xtest=xtest)
def main():
# hyperparameters
epochs = 1000
batches = 1024
z_k = 256
initializer = uniform_initializer(0.0001)
srng = RandomStreams(123)
opt = Adam(5e-7)
activation = leaky_relu
depth = 1
units = 1024
embedding_units = 128
# build and train
mlp_p = MLP(input_units=units,
output_units=z_k,
hidden_units=units,
hidden_depth=depth,
hidden_activation=activation,
output_activation=softmax_nd)
mlp_h = MLP(input_units=units,
output_units=units,
hidden_units=units,
hidden_depth=depth,
hidden_activation=activation)
outputpath = "output/skipgram_256_reinforce_highway"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
parameterization = HighwayParameterization(x_k=cooccurrence.shape[0],
z_k=z_k,
mlp_p=mlp_p,
mlp_h=mlp_h,
initializer=initializer,
activation=activation,
embedding_units=embedding_units,
units=units,
srng=srng)
model = ReinforceModel(cooccurrence=cooccurrence,
parameterization=parameterization,
z_k=z_k,
opt=opt)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
output_path = 'output/lstm_softmax_sparse_128_.05'
epochs = 100
batches = 5000
units = 1024
depth = 35
batch_size = 64
input_droput = 0.1
zoneout = 0.5
output_dropout = 0.5
regularizer = l2(1e-6)
activity_reg = 2.
temporal_activity_reg = 1.
layers = 2
initializer = uniform_initializer(0.05)
opt = Adam(1e-3)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
encoding = np.load('output/random_encodings/units-128/p-0.05/iter-0.npy')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
model = LSTMSoftmaxSparse(units=units,
vocab=vocab,
opt=opt,
zoneout=zoneout,
srng=srng,
encoding=encoding,
layers=layers,
regularizer=regularizer,
activity_reg=activity_reg,
temporal_activity_reg=temporal_activity_reg,
input_droput=input_droput,
output_dropout=output_dropout,
initializer=initializer)
model.train(output_path=output_path,
epochs=epochs,
batches=batches,
depth=depth,
batch_size=batch_size,
xtrain=xtrain,
xvalid=xvalid,
xtest=xtest)
def main():
epochs = 10
batches = 4096
z_k = 256
initializer = uniform_initializer(0.05)
outputpath = "output/skipgram_256_nn"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
initial_b = np.log(np.sum(cooccurrence, axis=0))
opt = Adam(1e-3)
model = FlatNNModel(
cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
initial_b=initial_b
)
model.train(outputpath=outputpath,
epochs=epochs,
batches=batches)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_reinforce_factored"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(5)
reg = EntropyRegularizer(1e-5)
model = ReinforceFactoredModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def sparse_speedtest(output_path, encoding):
iters = 1000
units = 512
depth = 35
batch_size = 64
input_droput = 0.
zoneout = 0.5
output_dropout = 0.
activity_reg = 0.
temporal_activity_reg = 0.
layers = 1
initializer = uniform_initializer(0.05)
opt = Adam(1e-3)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
i1 = np.arange(batch_size).reshape((-1, 1))
i2 = np.arange(depth).reshape((1, -1))
idx = i1 + i2
xb = xtrain[idx]
model = LSTMSoftmaxSparse(units=units,
encoding=encoding,
vocab=vocab,
opt=opt,
zoneout=zoneout,
srng=srng,
layers=layers,
activity_reg=activity_reg,
temporal_activity_reg=temporal_activity_reg,
input_droput=input_droput,
output_dropout=output_dropout,
initializer=initializer)
def train_fun():
model.train_fun(xb)
def test_fun():
model.nll_fun(xb)
return run_speedtest(output_path=output_path, train=train_fun, test=test_fun, iters=iters)
def main():
output_path = 'output/lstm_softmax_512'
epochs = 100
batches = 5000
units = 512
depth = 20
batch_size = 64
input_droput = 0.1
zoneout = 0.5
output_dropout = 0.5
regularizer = l2(1e-6)
activity_reg = 0
temporal_activity_reg = 0
layers = 2
initializer = uniform_initializer(0.05)
opt = Adam(1e-3)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
model = LSTMSoftmaxVanilla(units=units,
vocab=vocab,
opt=opt,
zoneout=zoneout,
srng=srng,
layers=layers,
regularizer=regularizer,
activity_reg=activity_reg,
temporal_activity_reg=temporal_activity_reg,
input_droput=input_droput,
output_dropout=output_dropout,
initializer=initializer)
model.train(output_path=output_path,
epochs=epochs,
batches=batches,
depth=depth,
batch_size=batch_size,
xtrain=xtrain,
xvalid=xvalid,
xtest=xtest)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_uniform"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(0.05)
reg = EntropyRegularizer(1e-6)
initial_b = np.log(np.sum(cooccurrence, axis=0))
model = UniformModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer,
initial_b=initial_b)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
output_path = 'output/lstm_softmax_vanilla_speedtest.npz'
iters = 1000
units = 512
depth = 35
batch_size = 64
zoneout = 0.
input_droput = 0.
output_dropout = 0.
initializer = uniform_initializer(0.05)
opt = Adam(1e-3)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
i1 = np.arange(batch_size).reshape((-1, 1))
i2 = np.arange(depth).reshape((1, -1))
idx = i1 + i2
xb = xtrain[idx]
model = LSTMSoftmaxVanilla(units=units,
vocab=vocab,
opt=opt,
zoneout=zoneout,
srng=srng,
input_droput=input_droput,
output_dropout=output_dropout,
initializer=initializer)
def train_fun():
model.train_fun(xb)
def test_fun():
model.nll_fun(xb)
run_speedtest(output_path=output_path,
train=train_fun,
test=test_fun,
iters=iters)
def main():
epochs = 1000
batches = 2048
batch_size = 64
tau0 = 5.
taurate = 3e-6
taumin = 0.5
output_path = '../output/mnist/gumbel_vae'
data = np.load('../output/mnist/mnist.npz')
xtrain, xtest = data["xtrain"], data["xtest"]
initializer = uniform_initializer(0.05)
K = 10 # number of classes
N = 20 # number of categorical distributions
encoder = MLP2(input_units=28 * 28,
units=[512, 256, K * N],
hidden_activation=T.nnet.relu,
initializer=initializer
)
decoder = MLP2(input_units=K * N,
units=[256, 512, 28 * 28],
hidden_activation=T.nnet.relu,
initializer=initializer
)
model = GumbelVaeModel(
encoder=encoder,
decoder=decoder,
opt=Adam(1e-3),
K=K,
N=N,
tau0=tau0,
taumin=taumin,
taurate=taurate
)
model.train(output_path=output_path,
epochs=epochs,
batches=batches,
batch_size=batch_size,
xtrain=xtrain,
xtest=xtest)
def main():
# hyperparameters
epochs = 1000
batches = 2048
z_k = 256
initializer = uniform_initializer(0.1)
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
units = 512
tao0 = 5.
tao_min = 0.1
tao_decay = 2e-4
mlp = MLP(input_units=z_k,
output_units=cooccurrence.shape[0],
hidden_units=units,
hidden_depth=1,
initializer=initializer,
use_bn=True,
hidden_activation=leaky_relu)
initial_b = np.log(np.sum(cooccurrence, axis=0))
K.set_value(mlp.bs[-1], initial_b)
pz_regularizer = EntropyRegularizer(3e-7)
# build and train
outputpath = "output/skipgram_256_gumbel3"
model = GumbelModel3(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mlp=mlp,
initializer=initializer,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 4096
z_k = 256
initializer = uniform_initializer(5)
srng = RandomStreams(123)
opt = Adam(1e-3)
# build and train
outputpath = "output/skipgram_256_reinforce_independent"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
parameterization = IndependentParameterization(x_k=cooccurrence.shape[0],
z_k=z_k,
initializer=initializer,
srng=srng)
model = ReinforceModel(cooccurrence=cooccurrence,
parameterization=parameterization,
z_k=z_k,
opt=opt)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
output_path = 'output/ganlm2'
epochs = 100
batches = 1000
val_batches = 1000
depth = 20
batch_size = 64
d_batches = 5
dopt = Adam(1e-3)
d_layers = 2
d_units = 512
d_input_dropout = 0.1
d_zoneout = 0.5
d_dropout = 0.5
d_regularizer = l2(1e-4)
gopt = Adam(1e-5)
g_layers = 2
g_units = 1024
g_input_dropout = 0.1
g_zoneout = 0.5
g_dropout = 0.5
g_regularizer = l2(1e-6)
regularizer_weight = 1e2
# constraint = clip_constraint(1e-1)
constraint = None
initializer = uniform_initializer(0.05)
srng = RandomStreams(123)
x = np.load('output/corpus/corpus.npz')
with open('output/corpus/vocab.pkl', 'rb') as f:
vocab = pickle.load(f)
xtrain, xvalid, xtest = x["train"], x["valid"], x["test"]
model = GANModel(vocab=vocab,
initializer=initializer,
dopt=dopt,
gopt=gopt,
srng=srng,
d_layers=d_layers,
d_units=d_units,
d_input_dropout=d_input_dropout,
d_dropout=d_dropout,
d_zoneout=d_zoneout,
g_layers=g_layers,
g_units=g_units,
g_input_dropout=g_input_dropout,
g_zoneout=g_zoneout,
g_dropout=g_dropout,
d_regularizer=d_regularizer,
g_regularizer=g_regularizer,
regularizer_weight=regularizer_weight,
constraint=constraint)
model.train(output_path=output_path,
epochs=epochs,
batches=batches,
depth=depth,
batch_size=batch_size,
val_batches=val_batches,
xtrain=xtrain,
xvalid=xvalid,
d_batches=d_batches,
xtest=xtest)
