def main():
tau0 = 2.
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_reinforce_smoothed"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(0.05)
reg = EntropyRegularizer(1e-5)
initial_b = np.log(np.sum(cooccurrence, axis=0))
initial_pz_weight = uniform_initializer(5.)((cooccurrence.shape[0], z_k))
model = ReinforceSmoothedModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer,
initial_pz_weight=initial_pz_weight,
initial_b=initial_b,
tau0=tau0)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
smoothing = 0.2
noise = 1.
opt = Adam(1e-3)
pz_regularizer = EntropyRegularizer(1e-5)
inputpath = 'output/skipgram_256-b/b-1.0e-04/iter-0/encodings-00000009.npy'
outputpath = "output/skipgram_256_finetune_reinforce_factored"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
# create initial weights
encoding = np.load(inputpath)
initial_pz = calc_initial_pz(encoding=encoding,
z_k=z_k,
smoothing=smoothing,
noise=noise)
model = ReinforceFactoredModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initial_pz_weight=initial_pz,
pz_regularizer=pz_regularizer,
initializer=None)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 4096
z_k = 256
initializer = uniform_initializer(0.05)
scale = 0.05
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
# initial_pz_weight = np.random.uniform(low=-scale, high=scale, size=(cooccurrence.shape[0], z_k))
opt = Adam(1e-3)
tao0 = 5.
tao_min = 0.25
tao_decay = 1e-5
pz_regularizer = EntropyRegularizer(3e-6)
# build and train
outputpath = "output/skipgram_256_gumbel1"
initial_b = np.log(np.sum(cooccurrence, axis=0))
model = GumbelModel1(
cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
initial_b=initial_b,
# initial_pz_weight=initial_pz_weight,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 2048
z_k = 256
initializer = uniform_initializer(0.05)
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
tao0 = 5.
tao_min = 0.1
tao_decay = 1e-4
pz_regularizer = EntropyRegularizer(3e-6)
# build and train
outputpath = "output/skipgram_256_gumbel2"
model = GumbelModel2(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_reinforce_factored"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(5)
reg = EntropyRegularizer(1e-5)
model = ReinforceFactoredModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 10
batches = 4096
z_k = 1024
iters = 3
outputpath = "output/skipgram_1024_h"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
weights = [1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7]
regularizers = [EntropyRegularizer(w) for w in weights]
labels = ["h-{:.01e}".format(w) for w in weights]
train_flat_regularizer_battery(outputpath=outputpath,
cooccurrence=cooccurrence,
epochs=epochs,
batches=batches,
iters=iters,
z_k=z_k,
labels=labels,
regularizers=regularizers,
is_weight_regularizer=False,
kwdata={'weights': np.array(weights)})
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_uniform"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(0.05)
reg = EntropyRegularizer(1e-6)
initial_b = np.log(np.sum(cooccurrence, axis=0))
model = UniformModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer,
initial_b=initial_b)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 2048
z_k = 256
initializer = uniform_initializer(0.1)
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
units = 512
tao0 = 5.
tao_min = 0.1
tao_decay = 2e-4
mlp = MLP(input_units=z_k,
output_units=cooccurrence.shape[0],
hidden_units=units,
hidden_depth=1,
initializer=initializer,
use_bn=True,
hidden_activation=leaky_relu)
initial_b = np.log(np.sum(cooccurrence, axis=0))
K.set_value(mlp.bs[-1], initial_b)
pz_regularizer = EntropyRegularizer(3e-7)
# build and train
outputpath = "output/skipgram_256_gumbel3"
model = GumbelModel3(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mlp=mlp,
initializer=initializer,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
opt = Adam(1e-3)
smoothing = 0.1
noise = 1
inputpath = 'output/skipgram_256-b/b-1.0e-04/iter-0/encodings-00000009.npy'
outputpath = "output/skipgram_256_finetune_reinforce_independent"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
pz_regularizer = EntropyRegularizer(1e-6)
# create initial weights
encoding = np.load(inputpath)
initial_pz = calc_initial_pz(encoding=encoding,
z_k=z_k,
smoothing=smoothing,
noise=noise)
srng = RandomStreams(123)
parameterization = IndependentParameterization(
x_k=cooccurrence.shape[0],
z_k=z_k,
initializer=None,
initial_pz_weight=initial_pz,
pz_regularizer=pz_regularizer,
srng=srng)
model = ReinforceModel(cooccurrence=cooccurrence,
parameterization=parameterization,
z_k=z_k,
opt=opt)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)