def main():
tau0 = 2.
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_reinforce_smoothed"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(0.05)
reg = EntropyRegularizer(1e-5)
initial_b = np.log(np.sum(cooccurrence, axis=0))
initial_pz_weight = uniform_initializer(5.)((cooccurrence.shape[0], z_k))
model = ReinforceSmoothedModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer,
initial_pz_weight=initial_pz_weight,
initial_b=initial_b,
tau0=tau0)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
smoothing = 0.2
noise = 1.
opt = Adam(1e-3)
pz_regularizer = EntropyRegularizer(1e-5)
inputpath = 'output/skipgram_256-b/b-1.0e-04/iter-0/encodings-00000009.npy'
outputpath = "output/skipgram_256_finetune_reinforce_factored"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
# create initial weights
encoding = np.load(inputpath)
initial_pz = calc_initial_pz(encoding=encoding,
z_k=z_k,
smoothing=smoothing,
noise=noise)
model = ReinforceFactoredModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initial_pz_weight=initial_pz,
pz_regularizer=pz_regularizer,
initializer=None)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 4096
z_k = 256
initializer = uniform_initializer(0.05)
scale = 0.05
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
# initial_pz_weight = np.random.uniform(low=-scale, high=scale, size=(cooccurrence.shape[0], z_k))
opt = Adam(1e-3)
tao0 = 5.
tao_min = 0.25
tao_decay = 1e-5
pz_regularizer = EntropyRegularizer(3e-6)
# build and train
outputpath = "output/skipgram_256_gumbel1"
initial_b = np.log(np.sum(cooccurrence, axis=0))
model = GumbelModel1(
cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
initial_b=initial_b,
# initial_pz_weight=initial_pz_weight,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 2048
z_k = 256
initializer = uniform_initializer(0.05)
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
tao0 = 5.
tao_min = 0.1
tao_decay = 1e-4
pz_regularizer = EntropyRegularizer(3e-6)
# build and train
outputpath = "output/skipgram_256_gumbel2"
model = GumbelModel2(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
initializer=initializer,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 1024
outputpath = "output/skipgram_flat_tst3"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
zpath = "output/z-00000016.npy"
enc = np.load(zpath)
reg = ExclusiveLasso(1e-10)
z = np.zeros((cooccurrence.shape[0], z_k), dtype='float32')
z[np.arange(z.shape[0]), enc] = 1
pz = np.dot(np.transpose(z, (1,0)), cooccurrence/np.sum(cooccurrence, axis=None))
m = np.sum(pz, axis=1, keepdims=True)
c = pz/m
h = np.sum(pz * -np.log(1e-9+c), axis=None)
print("H: {}".format(h))
scale = 1e-1
initial_pz = z * scale
opt = SGD(1e4)
model = FlatModel(cooccurrence=cooccurrence,
z_k=z_k,
pz_weight_regularizer=reg,
opt=opt,
initial_pz=initial_pz,
scale=scale)
model.train(outputpath,
epochs=epochs,
batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 1024
outputpath = "output/skipgram_flat_ngd"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e-2
opt = NGD(1e-3)
model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, scale=scale)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 1024
outputpath = "output/skipgram_flat_tst2"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e0
watchdog = Watchdog(limit=1e-6,
iters=500,
path="{}/watchdog.txt".format(outputpath))
opt = Adam(1e-3)
model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, scale=scale)
model.train(outputpath, epochs=epochs, watchdog=watchdog, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 1024
z_k = 256
initializer = uniform_initializer(0.0001)
srng = RandomStreams(123)
opt = Adam(5e-7)
activation = leaky_relu
depth = 1
units = 1024
embedding_units = 128
# build and train
mlp_p = MLP(input_units=units,
output_units=z_k,
hidden_units=units,
hidden_depth=depth,
hidden_activation=activation,
output_activation=softmax_nd)
mlp_h = MLP(input_units=units,
output_units=units,
hidden_units=units,
hidden_depth=depth,
hidden_activation=activation)
outputpath = "output/skipgram_256_reinforce_highway"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
parameterization = HighwayParameterization(x_k=cooccurrence.shape[0],
z_k=z_k,
mlp_p=mlp_p,
mlp_h=mlp_h,
initializer=initializer,
activation=activation,
embedding_units=embedding_units,
units=units,
srng=srng)
model = ReinforceModel(cooccurrence=cooccurrence,
parameterization=parameterization,
z_k=z_k,
opt=opt)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_reinforce_factored"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(5)
reg = EntropyRegularizer(1e-5)
model = ReinforceFactoredModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_m1"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e-2
opt = Adam(1e-3)
pz_regularizer = BalanceRegularizer(1e-7)
model = FlatModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mode=1,
pz_regularizer=pz_regularizer,
scale=scale)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_uniform"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
initializer = uniform_initializer(0.05)
reg = EntropyRegularizer(1e-6)
initial_b = np.log(np.sum(cooccurrence, axis=0))
model = UniformModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
pz_regularizer=reg,
initializer=initializer,
initial_b=initial_b)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 128
z_k = 256
outputpath = "output/skipgram_256_m2_el"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e-1
opt = AdamOptimizer(1e-3)
reg = ExclusiveLasso(1e-10)
model = FlatModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mode=2,
pz_weight_regularizer=reg,
scale=scale)
model.train(outputpath,
epochs=epochs,
batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 2048
z_k = 256
initializer = uniform_initializer(0.1)
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
opt = Adam(1e-3)
units = 512
tao0 = 5.
tao_min = 0.1
tao_decay = 2e-4
mlp = MLP(input_units=z_k,
output_units=cooccurrence.shape[0],
hidden_units=units,
hidden_depth=1,
initializer=initializer,
use_bn=True,
hidden_activation=leaky_relu)
initial_b = np.log(np.sum(cooccurrence, axis=0))
K.set_value(mlp.bs[-1], initial_b)
pz_regularizer = EntropyRegularizer(3e-7)
# build and train
outputpath = "output/skipgram_256_gumbel3"
model = GumbelModel3(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mlp=mlp,
initializer=initializer,
pz_regularizer=pz_regularizer,
tao0=tao0,
tao_min=tao_min,
tao_decay=tao_decay)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
opt = Adam(1e-3)
smoothing = 0.1
noise = 1
inputpath = 'output/skipgram_256-b/b-1.0e-04/iter-0/encodings-00000009.npy'
outputpath = "output/skipgram_256_finetune_reinforce_independent"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
pz_regularizer = EntropyRegularizer(1e-6)
# create initial weights
encoding = np.load(inputpath)
initial_pz = calc_initial_pz(encoding=encoding,
z_k=z_k,
smoothing=smoothing,
noise=noise)
srng = RandomStreams(123)
parameterization = IndependentParameterization(
x_k=cooccurrence.shape[0],
z_k=z_k,
initializer=None,
initial_pz_weight=initial_pz,
pz_regularizer=pz_regularizer,
srng=srng)
model = ReinforceModel(cooccurrence=cooccurrence,
parameterization=parameterization,
z_k=z_k,
opt=opt)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
# hyperparameters
epochs = 1000
batches = 4096
z_k = 256
initializer = uniform_initializer(5)
srng = RandomStreams(123)
opt = Adam(1e-3)
# build and train
outputpath = "output/skipgram_256_reinforce_independent"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
parameterization = IndependentParameterization(x_k=cooccurrence.shape[0],
z_k=z_k,
initializer=initializer,
srng=srng)
model = ReinforceModel(cooccurrence=cooccurrence,
parameterization=parameterization,
z_k=z_k,
opt=opt)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
