def pento(n_trials):
ri = numpy.random.random_integers
state = DD()
with open('mnist_powerup_temp.yaml') as ymtmp:
state.yaml_string = ymtmp.read()
state.powerup_nunits = 240
state.powerup_npieces = 5
state.W_lr_scale = 0.04
state.p_lr_scale = 0.01
state.lr_rate = 0.1
state.l2_pen = 1e-5
state.l2_pen2 = 0.0000
state.init_mom = 0.5
state.final_mom = 0.5
state.decay_factor = 0.5
state.max_col_norm = 1.9365
state.max_col_norm2 = 1.8365
state.batch_size = 128
state.save_path = './'
n_pieces = [2, 3, 4, 5, 6, 8, 10, 12, 14, 16]
n_units = [200, 240, 280, 320, 360, 420, 480]
batch_sizes = [128, 256, 512]
learning_rates = numpy.logspace(numpy.log10(0.001), numpy.log10(1.0), 30)
learning_rate_scalers = numpy.logspace(numpy.log10(0.01), numpy.log10(1), 30)
l2_pen = numpy.logspace(numpy.log10(1e-6), numpy.log10(8*1e-3), 100)
max_col_norms = [1.7365, 1.8365, 1.9365, 2.1365, 2.2365, 2.4365]
ind = 0
TABLE_NAME = "powerup_mnist_1layer_fixed"
db = api0.open_db('postgresql://[email protected]/gulcehrc_db?table=' + TABLE_NAME)
for i in xrange(n_trials):
state.lr_rate = learning_rates[ri(learning_rates.shape[0]) - 1]
state.powerup_nunits = n_units[ri(len(n_units)) - 1]
state.powerup_npieces = n_pieces[ri(len(n_pieces)) - 1]
state.W_lr_scale = learning_rate_scalers[ri(len(learning_rate_scalers)) - 1]
state.p_lr_scale = learning_rate_scalers[ri(len(learning_rate_scalers)) - 1]
state.batch_size = batch_sizes[ri(len(batch_sizes)) - 1]
state.l2_pen = l2_pen[ri(l2_pen.shape[0]) - 1]
state.init_mom = numpy.random.uniform(low=0.3, high=0.6)
state.final_mom = numpy.random.uniform(low=state.init_mom + 0.1, high=0.9)
state.decay_factor = numpy.random.uniform(low=0.01, high=0.05)
state.max_col_norm = max_col_norms[ri(len(max_col_norms)) - 1]
alphabet = list('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789')
numpy.random.shuffle(alphabet)
state.save_path = './'
state.save_path += ''.join(alphabet[:7]) + '_'
sql.insert_job(experiment, flatten(state), db)
ind += 1
db.createView(TABLE_NAME + '_view')
print "{} jobs submitted".format(ind)
def update_default_layer_hyperparams(state):
#state = sanity_checks(state)
prev_layer_dim = get_dim_input(state)
for key,layer in state.layers.iteritems():
default_hyperparams = copy.deepcopy(layer_config)
update_irange_in_layer(layer, prev_layer_dim)
prev_layer_dim = layer.dim
default_hyperparams.update(layer)
state.layers[key] = default_hyperparams
# TODO: find a better way to check for list of items in the hyperparams.
# random_sampling should give a good string representation for list of values
# in the hyperparams.
flattened_state = flatten(state)
for k,v in flattened_state.iteritems():
if str(v)[0] == '[' and str(v)[-1] == ']' and type(v) != type([]):
# It is a list of values but represented as a string.
# Get just the content of the list.
values = str(v)[1:-1]
# Add the content in a proper list (not represented as a string).
values = [val.strip() for val in values.split(',')]
flattened_state[k] = values
# TODO : why the modified state is not kept after returning from this function?
# However, the updated state above is kept after returning from this function.
state = expand(flattened_state)
return state
def update_default_layer_hyperparams(state):
#state = sanity_checks(state)
prev_layer_dim = get_dim_input(state)
for key, layer in state.layers.iteritems():
default_hyperparams = copy.deepcopy(layer_config)
update_irange_in_layer(layer, prev_layer_dim)
prev_layer_dim = layer.dim
default_hyperparams.update(layer)
state.layers[key] = default_hyperparams
# TODO: find a better way to check for list of items in the hyperparams.
# random_sampling should give a good string representation for list of values
# in the hyperparams.
flattened_state = flatten(state)
for k, v in flattened_state.iteritems():
if str(v)[0] == '[' and str(v)[-1] == ']' and type(v) != type([]):
# It is a list of values but represented as a string.
# Get just the content of the list.
values = str(v)[1:-1]
# Add the content in a proper list (not represented as a string).
values = [val.strip() for val in values.split(',')]
flattened_state[k] = values
# TODO : why the modified state is not kept after returning from this function?
# However, the updated state above is kept after returning from this function.
state = expand(flattened_state)
return state
def tfd(n_trials):
ri = numpy.random.random_integers
state = DD()
with open('mnist_powerup_temp_l2.yaml') as ymtmp:
state.yaml_string = ymtmp.read()
state.powerup_nunits = 240
state.powerup_npieces = 5
state.powerup_nunits2 = 240
state.powerup_npieces2 = 5
state.W_lr_scale = 0.04
state.p_lr_scale = 0.01
state.lr_rate = 0.1
state.init_mom = 0.5
state.final_mom = 0.5
state.decay_factor = 0.5
state.max_col_norm = 1.9365
state.save_path = './'
n_pieces = [2, 3, 4, 5]
n_units = [200, 240, 320, 360, 420, 480]
learning_rates = numpy.logspace(numpy.log10(0.09), numpy.log10(1.2), 60)
learning_rate_scalers = numpy.logspace(numpy.log10(0.1), numpy.log10(1), 50)
decay_factors = numpy.logspace(numpy.log10(0.001), numpy.log10(0.06), 40)
max_col_norms = [1.8365, 1.9365, 2.1365, 2.2365, 2.3486]
ind = 0
TABLE_NAME = "powerup_mnist_finest_large_2l"
db = api0.open_db('postgresql://[email protected]/gulcehrc_db?table=' + TABLE_NAME)
for i in xrange(n_trials):
state.lr_rate = learning_rates[ri(learning_rates.shape[0]) - 1]
state.powerup_nunits = n_units[ri(len(n_units)) - 1]
state.powerup_npieces = n_pieces[ri(len(n_pieces) - 1)]
state.powerup_nunits2 = state.powerup_nunits
state.powerup_npieces2 = state.powerup_npieces
state.W_lr_scale = numpy.random.uniform(low=0.09, high=1.0)
state.p_lr_scale = numpy.random.uniform(low=0.09, high=1.0)
state.init_mom = numpy.random.uniform(low=0.3, high=0.6)
state.final_mom = numpy.random.uniform(low=state.init_mom + 0.1, high=0.9)
state.decay_factor = decay_factors[ri(len(decay_factors)) - 1]
state.max_col_norm = max_col_norms[ri(len(max_col_norms)) - 1]
alphabet = list('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789')
state.save_path = './'
state.save_path += ''.join(alphabet[:7]) + '_'
sql.insert_job(experiment, flatten(state), db)
ind += 1
db.createView(TABLE_NAME + '_view')
print "{} jobs submitted".format(ind)
def tfd(n_trials):
ri = numpy.random.random_integers
state = DD()
with open("tfd_powerup_temp.yaml") as ymtmp:
state.yaml_string = ymtmp.read()
state.powerup_nunits = 240
state.powerup_npieces = 5
state.W_lr_scale = 0.04
state.p_lr_scale = 0.01
state.lr_rate = 0.1
state.l2_pen = 1e-5
state.l2_pen2 = 0.0000
state.init_mom = 0.5
state.final_mom = 0.5
state.decay_factor = 0.5
state.max_col_norm = 1.9365
state.max_col_norm2 = 1.8365
state.save_path = "./"
n_pieces = [2, 3, 4, 5, 6, 8, 10, 12, 14, 16]
n_units = [200, 240, 280, 320, 420]
learning_rates = numpy.logspace(numpy.log10(0.001), numpy.log10(1.0), 32)
learning_rate_scalers = numpy.logspace(numpy.log10(0.01), numpy.log10(1), 30)
l2_pen = numpy.logspace(numpy.log10(1e-6), numpy.log10(3 * 1e-3), 100)
max_col_norms = [1.8365, 1.9365, 2.1365, 2.2365, 2.3486, 2.4365]
ind = 0
TABLE_NAME = "powerup_tfd_1layer_finer_large2"
db = api0.open_db("postgresql://[email protected]/gulcehrc_db?table=" + TABLE_NAME)
for i in xrange(n_trials):
state.lr_rate = learning_rates[ri(learning_rates.shape[0]) - 1]
state.powerup_nunits = n_units[ri(len(n_units)) - 1]
if state.powerup_nunits >= 320:
state.powerup_npieces = n_pieces[ri(low=0, high=5)]
else:
state.powerup_npieces = n_pieces[ri(low=3, high=(len(n_pieces) - 1))]
state.W_lr_scale = learning_rate_scalers[ri(len(learning_rate_scalers)) - 1]
state.p_lr_scale = learning_rate_scalers[ri(len(learning_rate_scalers)) - 1]
state.l2_pen = l2_pen[ri(l2_pen.shape[0]) - 1]
state.init_mom = numpy.random.uniform(low=0.3, high=0.6)
state.final_mom = numpy.random.uniform(low=state.init_mom + 1.0, high=0.9)
state.decay_factor = numpy.random.uniform(low=0.01, high=0.05)
state.max_col_norm = max_col_norms[ri(len(max_col_norms)) - 1]
alphabet = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789")
state.save_path = "./"
state.save_path += "".join(alphabet[:7]) + "_"
sql.insert_job(experiment, flatten(state), db)
ind += 1
db.createView(TABLE_NAME + "_view")
print "{} jobs submitted".format(ind)
def generate_jobs_list(n_jobs, recorded, use_gpu, host, model_config):
exps_list = []
for i in range(n_jobs):
# TODO: do not hardcode the common options!
if recorded:
print('..outputs of job (' + str(i) + ') will be recorded')
exp_cmd = 'jobman -r cmdline experiment.job '
else:
exp_cmd = 'jobman cmdline experiment.job '
if type(use_gpu) is int and host is 'local':
exp_cmd = 'THEANO_FLAGS=device=gpu%d,floatX=float32 '%use_gpu + exp_cmd
exp_cmd = cmd_line_embed(exp_cmd, flatten(model_config))
# print exp_cmd
exps_list.append(exp_cmd)
return exps_list
state.mem_nel = 200
state.mem_size = 28
np.random.seed(3)
ri = np.random.random_integers
learning_rates = np.logspace(np.log10(lr_min), np.log10(lr_max), 100)
stds = np.random.uniform(std_min, std_max, 100)
#Change the table name everytime you try
TABLE_NAME = "adam_grusoft_model_search_v0"
# You should have an account for jobman
db = api0.open_db('postgresql://[email protected]/gulcehrc_db?table=' + TABLE_NAME)
ind = 0
for i in xrange(n_trials):
state.lr = learning_rates[ri(learning_rates.shape[0]) - 1]
state.std = stds[ri(len(stds)) - 1]
state.batch_size = batches[ri(len(batches)) - 1]
state.renormalization_scale = renormalization_scale[ri(len(renormalization_scale)) - 1]
state.mem_nel = mem_nels[ri(len(mem_nels)) - 1]
state.mem_size = mem_sizes[ri(len(mem_sizes)) - 1]
state.std = stds[ri(stds.shape[0]) - 1]
sql.insert_job(memnet.train_model_adam_gru_soft.search_model_adam_gru_soft, flatten(state), db)
ind += 1
db.createView(TABLE_NAME + "_view")
print "{} jobs submitted".format(ind)
for i in range(args.n_jobs):
# TODO: do not hardcode the common options!
if args.record:
print('..outputs of job (' + str(i) + ') will be recorded')
exp_cmd = 'jobman -r cmdline %s '%exp_model
else:
exp_cmd = 'jobman cmdline %s '%exp_model
print exp_cmd
if 'ip05' in host:
exp_cmd = 'THEANO_FLAGS=floatX=float32 ' + exp_cmd
if args.use_gpu and host == 'local':
exp_cmd = 'THEANO_FLAGS=device=gpu,floatX=float32 ' + exp_cmd
exp_cmd = cmd_line_embed(exp_cmd, flatten(model_config[model]))
f.write(exp_cmd+'\n')
exps_by_model.setdefault(model, [])
exps_by_model[model].append(exp_cmd)
f.close()
os.chdir(jobs_folder)
print '..commands: ', cmd
if not args.n_concur_jobs:
os.system(cmd)
else:
print 'Jobs will be run locally.'
print '%s jobs will be run simultaneously.'%args.n_concur_jobs
for i in range(args.n_jobs):
# TODO: do not hardcode the common options!
#exp_cmd = 'jobman -r cmdline mlp_training_framework.model.experimTHEANO_FLAGS=profile=True,floatX=float32ent '
if args.record:
exp_cmd = 'jobman -r cmdline mlp_model.model.experiment '
else:
exp_cmd = 'jobman cmdline mlp_model.model.experiment '
print exp_cmd
if 'ip05' in host:
exp_cmd = 'THEANO_FLAGS=floatX=float32 ' + exp_cmd
if args.use_gpu and host == 'local':
exp_cmd = 'THEANO_FLAGS=device=gpu2,floatX=float32 ' + exp_cmd
exp_cmd = cmd_line_embed(exp_cmd, flatten(model_config[model]))
f.write(exp_cmd+'\n')
exps_by_model.setdefault(model, [])
exps_by_model[model].append(exp_cmd)
#import pdb
#pdb.set_trace()
f.close()
#import pdb
#pdb.set_trace()
#print '====cmd====', cmd
os.chdir(model)
#print '==args.n_concur_jobs', args.n_concur_jobs
if not args.n_concur_jobs:
#import pdb
