def main():
epochs = 1000
batches = 4096
z_k = 1024
outputpath = "output/skipgram_flat_tst3"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
zpath = "output/z-00000016.npy"
enc = np.load(zpath)
reg = ExclusiveLasso(1e-10)
z = np.zeros((cooccurrence.shape[0], z_k), dtype='float32')
z[np.arange(z.shape[0]), enc] = 1
pz = np.dot(np.transpose(z, (1,0)), cooccurrence/np.sum(cooccurrence, axis=None))
m = np.sum(pz, axis=1, keepdims=True)
c = pz/m
h = np.sum(pz * -np.log(1e-9+c), axis=None)
print("H: {}".format(h))
scale = 1e-1
initial_pz = z * scale
opt = SGD(1e4)
model = FlatModel(cooccurrence=cooccurrence,
z_k=z_k,
pz_weight_regularizer=reg,
opt=opt,
initial_pz=initial_pz,
scale=scale)
model.train(outputpath,
epochs=epochs,
batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 1024
outputpath = "output/skipgram_flat_ngd"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e-2
opt = NGD(1e-3)
model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, scale=scale)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 1024
outputpath = "output/skipgram_flat_tst2"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e0
watchdog = Watchdog(limit=1e-6,
iters=500,
path="{}/watchdog.txt".format(outputpath))
opt = Adam(1e-3)
model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, scale=scale)
model.train(outputpath, epochs=epochs, watchdog=watchdog, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 4096
z_k = 256
outputpath = "output/skipgram_256_m1"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e-2
opt = Adam(1e-3)
pz_regularizer = BalanceRegularizer(1e-7)
model = FlatModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mode=1,
pz_regularizer=pz_regularizer,
scale=scale)
model.train(outputpath, epochs=epochs, batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(
outputpath, "validate.txt"),
cooccurrence=cooccurrence)
def main():
epochs = 1000
batches = 128
z_k = 256
outputpath = "output/skipgram_256_m2_el"
cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32)
scale = 1e-1
opt = AdamOptimizer(1e-3)
reg = ExclusiveLasso(1e-10)
model = FlatModel(cooccurrence=cooccurrence,
z_k=z_k,
opt=opt,
mode=2,
pz_weight_regularizer=reg,
scale=scale)
model.train(outputpath,
epochs=epochs,
batches=batches)
return run_flat_validation(input_path=outputpath,
output_path=os.path.join(outputpath, "validate.txt"),
cooccurrence=cooccurrence)