def main(): tau0 = 2. epochs = 1000 batches = 4096 z_k = 256 outputpath = "output/skipgram_256_reinforce_smoothed" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) opt = Adam(1e-3) initializer = uniform_initializer(0.05) reg = EntropyRegularizer(1e-5) initial_b = np.log(np.sum(cooccurrence, axis=0)) initial_pz_weight = uniform_initializer(5.)((cooccurrence.shape[0], z_k)) model = ReinforceSmoothedModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, pz_regularizer=reg, initializer=initializer, initial_pz_weight=initial_pz_weight, initial_b=initial_b, tau0=tau0) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 256 smoothing = 0.2 noise = 1. opt = Adam(1e-3) pz_regularizer = EntropyRegularizer(1e-5) inputpath = 'output/skipgram_256-b/b-1.0e-04/iter-0/encodings-00000009.npy' outputpath = "output/skipgram_256_finetune_reinforce_factored" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) # create initial weights encoding = np.load(inputpath) initial_pz = calc_initial_pz(encoding=encoding, z_k=z_k, smoothing=smoothing, noise=noise) model = ReinforceFactoredModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, initial_pz_weight=initial_pz, pz_regularizer=pz_regularizer, initializer=None) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): # hyperparameters epochs = 1000 batches = 4096 z_k = 256 initializer = uniform_initializer(0.05) scale = 0.05 cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) # initial_pz_weight = np.random.uniform(low=-scale, high=scale, size=(cooccurrence.shape[0], z_k)) opt = Adam(1e-3) tao0 = 5. tao_min = 0.25 tao_decay = 1e-5 pz_regularizer = EntropyRegularizer(3e-6) # build and train outputpath = "output/skipgram_256_gumbel1" initial_b = np.log(np.sum(cooccurrence, axis=0)) model = GumbelModel1( cooccurrence=cooccurrence, z_k=z_k, opt=opt, initializer=initializer, initial_b=initial_b, # initial_pz_weight=initial_pz_weight, pz_regularizer=pz_regularizer, tao0=tao0, tao_min=tao_min, tao_decay=tao_decay) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): # hyperparameters epochs = 1000 batches = 2048 z_k = 256 initializer = uniform_initializer(0.05) cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) opt = Adam(1e-3) tao0 = 5. tao_min = 0.1 tao_decay = 1e-4 pz_regularizer = EntropyRegularizer(3e-6) # build and train outputpath = "output/skipgram_256_gumbel2" model = GumbelModel2(cooccurrence=cooccurrence, z_k=z_k, opt=opt, initializer=initializer, pz_regularizer=pz_regularizer, tao0=tao0, tao_min=tao_min, tao_decay=tao_decay) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 1024 outputpath = "output/skipgram_flat_tst3" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) zpath = "output/z-00000016.npy" enc = np.load(zpath) reg = ExclusiveLasso(1e-10) z = np.zeros((cooccurrence.shape[0], z_k), dtype='float32') z[np.arange(z.shape[0]), enc] = 1 pz = np.dot(np.transpose(z, (1,0)), cooccurrence/np.sum(cooccurrence, axis=None)) m = np.sum(pz, axis=1, keepdims=True) c = pz/m h = np.sum(pz * -np.log(1e-9+c), axis=None) print("H: {}".format(h)) scale = 1e-1 initial_pz = z * scale opt = SGD(1e4) model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, pz_weight_regularizer=reg, opt=opt, initial_pz=initial_pz, scale=scale) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join(outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 1024 outputpath = "output/skipgram_flat_ngd" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) scale = 1e-2 opt = NGD(1e-3) model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, scale=scale) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 1024 outputpath = "output/skipgram_flat_tst2" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) scale = 1e0 watchdog = Watchdog(limit=1e-6, iters=500, path="{}/watchdog.txt".format(outputpath)) opt = Adam(1e-3) model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, scale=scale) model.train(outputpath, epochs=epochs, watchdog=watchdog, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): # hyperparameters epochs = 1000 batches = 1024 z_k = 256 initializer = uniform_initializer(0.0001) srng = RandomStreams(123) opt = Adam(5e-7) activation = leaky_relu depth = 1 units = 1024 embedding_units = 128 # build and train mlp_p = MLP(input_units=units, output_units=z_k, hidden_units=units, hidden_depth=depth, hidden_activation=activation, output_activation=softmax_nd) mlp_h = MLP(input_units=units, output_units=units, hidden_units=units, hidden_depth=depth, hidden_activation=activation) outputpath = "output/skipgram_256_reinforce_highway" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) parameterization = HighwayParameterization(x_k=cooccurrence.shape[0], z_k=z_k, mlp_p=mlp_p, mlp_h=mlp_h, initializer=initializer, activation=activation, embedding_units=embedding_units, units=units, srng=srng) model = ReinforceModel(cooccurrence=cooccurrence, parameterization=parameterization, z_k=z_k, opt=opt) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 256 outputpath = "output/skipgram_256_reinforce_factored" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) opt = Adam(1e-3) initializer = uniform_initializer(5) reg = EntropyRegularizer(1e-5) model = ReinforceFactoredModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, pz_regularizer=reg, initializer=initializer) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 256 outputpath = "output/skipgram_256_m1" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) scale = 1e-2 opt = Adam(1e-3) pz_regularizer = BalanceRegularizer(1e-7) model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, mode=1, pz_regularizer=pz_regularizer, scale=scale) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 256 outputpath = "output/skipgram_256_uniform" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) opt = Adam(1e-3) initializer = uniform_initializer(0.05) reg = EntropyRegularizer(1e-6) initial_b = np.log(np.sum(cooccurrence, axis=0)) model = UniformModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, pz_regularizer=reg, initializer=initializer, initial_b=initial_b) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 128 z_k = 256 outputpath = "output/skipgram_256_m2_el" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) scale = 1e-1 opt = AdamOptimizer(1e-3) reg = ExclusiveLasso(1e-10) model = FlatModel(cooccurrence=cooccurrence, z_k=z_k, opt=opt, mode=2, pz_weight_regularizer=reg, scale=scale) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join(outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): # hyperparameters epochs = 1000 batches = 2048 z_k = 256 initializer = uniform_initializer(0.1) cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) opt = Adam(1e-3) units = 512 tao0 = 5. tao_min = 0.1 tao_decay = 2e-4 mlp = MLP(input_units=z_k, output_units=cooccurrence.shape[0], hidden_units=units, hidden_depth=1, initializer=initializer, use_bn=True, hidden_activation=leaky_relu) initial_b = np.log(np.sum(cooccurrence, axis=0)) K.set_value(mlp.bs[-1], initial_b) pz_regularizer = EntropyRegularizer(3e-7) # build and train outputpath = "output/skipgram_256_gumbel3" model = GumbelModel3(cooccurrence=cooccurrence, z_k=z_k, opt=opt, mlp=mlp, initializer=initializer, pz_regularizer=pz_regularizer, tao0=tao0, tao_min=tao_min, tao_decay=tao_decay) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): epochs = 1000 batches = 4096 z_k = 256 opt = Adam(1e-3) smoothing = 0.1 noise = 1 inputpath = 'output/skipgram_256-b/b-1.0e-04/iter-0/encodings-00000009.npy' outputpath = "output/skipgram_256_finetune_reinforce_independent" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) pz_regularizer = EntropyRegularizer(1e-6) # create initial weights encoding = np.load(inputpath) initial_pz = calc_initial_pz(encoding=encoding, z_k=z_k, smoothing=smoothing, noise=noise) srng = RandomStreams(123) parameterization = IndependentParameterization( x_k=cooccurrence.shape[0], z_k=z_k, initializer=None, initial_pz_weight=initial_pz, pz_regularizer=pz_regularizer, srng=srng) model = ReinforceModel(cooccurrence=cooccurrence, parameterization=parameterization, z_k=z_k, opt=opt) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)
def main(): # hyperparameters epochs = 1000 batches = 4096 z_k = 256 initializer = uniform_initializer(5) srng = RandomStreams(123) opt = Adam(1e-3) # build and train outputpath = "output/skipgram_256_reinforce_independent" cooccurrence = np.load('output/cooccurrence.npy').astype(np.float32) parameterization = IndependentParameterization(x_k=cooccurrence.shape[0], z_k=z_k, initializer=initializer, srng=srng) model = ReinforceModel(cooccurrence=cooccurrence, parameterization=parameterization, z_k=z_k, opt=opt) model.train(outputpath, epochs=epochs, batches=batches) return run_flat_validation(input_path=outputpath, output_path=os.path.join( outputpath, "validate.txt"), cooccurrence=cooccurrence)