def update_performance(
experiment_id,
experiment,
train_score,
train_loss,
val_score,
val_loss,
step,
num_params,
ckpt_path,
results_path,
summary_path):
"""Update performance table for an experiment."""
config = credentials.postgresql_connection()
perf_dict = {
'experiment_id': experiment_id,
'experiment': experiment,
'train_score': train_score,
'train_loss': train_loss,
'val_score': val_score,
'val_loss': val_loss,
'step': step,
'num_params': num_params,
'ckpt_path': ckpt_path,
'results_path': results_path,
'summary_path': summary_path,
}
with db(config) as db_conn:
db_conn.update_performance(perf_dict)
def query_hp_hist(exp_params, eval_on='validation_loss', init_top=1e10):
"""Query an experiment's history of hyperparameters and performance."""
config = credentials.postgresql_connection()
domain_param_map = hp_opt_utils.hp_opt_dict()
experiment_name = exp_params['experiment_name']
with db(config) as db_conn:
perf = db_conn.get_performance(experiment_name=experiment_name)
if len(perf) == 0:
# And set hp history to initial values.
hp_history = {}
for k, v in domain_param_map.iteritems():
if exp_params[k] is not None: # If searching this domain.
hp_history[v] = exp_params[v]
# First step of hp-optim. Requires > 1 entry for X/Y.
perf = [
[init_top + ((np.random.rand() - 0.5) * init_top / 10)],
[init_top + ((np.random.rand() - 0.5) * init_top / 10)]
]
hp_history = [
hp_history,
hp_history
]
else:
# Sort performance by time elapsed
times = [x['time_elapsed'] for x in perf]
time_idx = np.argsort(times)
perf = [perf[idx][eval_on] for idx in time_idx]
# Sort hp parameters by history
times = [x['experiment_iteration'] for x in exp_params]
time_idx = np.argsort(times)
hp_history = [exp_params[idx] for idx in time_idx]
return perf, hp_history
def get_performance(experiment_name, force_fwd=False):
"""Get performance for an experiment."""
config = credentials.postgresql_connection()
if force_fwd:
config.db_ssh_forward = True
with db(config) as db_conn:
perf = db_conn.get_performance(experiment_name=experiment_name)
return perf
def get_cells_all_data_by_rf(list_of_dicts):
"""Get all data for cells by their RF centers."""
config = credentials.postgresql_connection()
queries = []
with data_db(config) as db_conn:
for d in list_of_dicts:
queries += [db_conn.gather_data_by_rf_coor(d)]
return queries
def get_experiment_name():
"""Get names of experiments."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
param_dict = db_conn.get_parameters()
if param_dict is None:
print 'No remaining experiments to run.'
sys.exit(1)
return param_dict['experiment_name']
def get_cells_all_data_by_rf_and_stimuli(rfs, stimuli, sessions=None):
"""Get all data for cells by their RF centers."""
config = credentials.postgresql_connection()
queries = []
with data_db(config) as db_conn:
for it_rf in rfs:
queries += [
db_conn.gather_data_by_rf_coor_and_stim(
it_rf, stimuli, sessions)
]
return queries
def query_hp_hist(exp_params,
eval_on='validation_loss',
performance_metric='validation_loss',
aggregator='max'):
"""Query an experiment's history of hyperparameters and performance."""
config = credentials.postgresql_connection()
experiment_link = exp_params['experiment_link']
with db(config) as db_conn:
perf = db_conn.get_performance_by_experiment_link(
experiment_link=experiment_link,
performance_metric=performance_metric,
aggregator=aggregator)
assert perf is not None, 'Error: No performance history found.'
return perf
def get_parameters_evaluation(experiment_name, log, random=False):
"""Get parameters for a given experiment."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
param_dict = db_conn.get_parameters_for_evaluation(
experiment_name=experiment_name,
random=random)
log.info('Using parameters: %s' % json.dumps(param_dict, indent=4))
if param_dict is not None:
experiment_id = param_dict['_id']
else:
experiment_id = None
if param_dict is None:
raise RuntimeError('This experiment is complete.')
return param_dict, experiment_id
def update_performance(experiment_id, experiment_name, summary_dir, ckpt_file,
training_loss, validation_loss, time_elapsed,
training_step):
"""Update performance table for an experiment."""
config = credentials.postgresql_connection()
perf_dict = {
'experiment_id': experiment_id,
'experiment_name': experiment_name,
'summary_dir': summary_dir,
'ckpt_file': ckpt_file,
'training_loss': training_loss,
'validation_loss': validation_loss,
'time_elapsed': time_elapsed,
'training_step': training_step,
}
with db(config) as db_conn:
db_conn.update_performance(perf_dict)
def __enter__(self):
if main_config.db_ssh_forward:
forward = sshtunnel.SSHTunnelForwarder(
credentials.machine_credentials()['ssh_address'],
ssh_username=credentials.machine_credentials()['username'],
ssh_password=credentials.machine_credentials()['password'],
remote_bind_address=('127.0.0.1', 5432))
forward.start()
self.forward = forward
self.pgsql_port = forward.local_bind_port
else:
self.forward = None
self.pgsql_port = ''
pgsql_string = credentials.postgresql_connection(str(self.pgsql_port))
self.pgsql_string = pgsql_string
self.conn = psycopg2.connect(**pgsql_string)
self.conn.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
self.cur = self.conn.cursor(
cursor_factory=psycopg2.extras.RealDictCursor)
return self
def add_cell_data(
cell_rf_dict,
list_of_cell_stim_dicts):
"""Add a cell to the databse.
Inputs:::
cell_rf_dict: dictionary containing cell_id_number and its RF properties.
list_of_cell_stim_dicts: a list of dictionaries, each containing the cell's
id + a pointer to a data numpy file and a boolean for the stimuli it
contains.
------------------------------
For a given cell, e.g., cell_1
cell_rf_dict = {
'cell_id': 1,
'rf': big
}
list_of_cell_stim_dicts[
{
'cell_id': 1,
'session': A,
'drifting_gratings': True,
'ALL OTHER COLUMNS': False,
'cell_npy': os.path.join(data_dir, '%s_%s_%s.npy' % (
cell_id, session, stimulus))
},
{
'cell_id': 1,
'session': B,
'drifting_gratings': True,
'ALL OTHER COLUMNS': False,
'cell_npy': os.path.join(
data_dir, '%s_%s_%s.npy' % (cell_id, session, stimulus))
}
]
"""
config = credentials.postgresql_connection()
with data_db(config) as db_conn:
db_conn.populate_db_with_rf([cell_rf_dict])
db_conn.populate_db_with_cell_stim(list_of_cell_stim_dicts)
def get_parameters(experiment_name, log, random=False, evaluation=None):
"""Get parameters for a given experiment."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
if evaluation is None:
param_dict = db_conn.get_parameters_and_reserve(
experiment_name=experiment_name, random=random)
else:
ckpt_name = evaluation.split('-')[0]
param_dict = db_conn.get_parameters_for_evaluation(
experiment_name=experiment_name,
ckpt_name=ckpt_name,
random=random)
log.info('Using parameters: %s' % json.dumps(param_dict, indent=4))
if param_dict is not None:
experiment_id = param_dict['_id']
# db_conn.update_in_process(
# experiment_id=experiment_id,
# experiment_name=experiment_name)
else:
experiment_id = None
if param_dict is None:
raise RuntimeError('This experiment is complete.')
return param_dict, experiment_id
def initialize_database():
"""Initialize the psql database from the schema file."""
config = credentials.postgresql_connection()
with data_db(config) as db_conn:
db_conn.recreate_db(run=True)
db_conn.return_status('CREATE')
def get_performance(experiment_name):
"""Get performance for an experiment."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
perf = db_conn.get_performance(experiment_name=experiment_name)
return perf
def list_experiments():
"""List all experiments in the database."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
experiments = db_conn.list_experiments()
return experiments
def reset_in_process():
"""Reset the in_process table."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
db_conn.reset_in_process()
print 'Cleared the in_process table.'
def initialize_database():
"""Initialize and recreate the database."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
db_conn.recreate_db(run=True)
db_conn.return_status('CREATE')
def get_summary_list(experiment_name):
"""Get list of tensorboard summaries."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
summaries = db_conn.get_summaries(experiment_name)
return summaries
def get_performance(experiment_name):
config = credentials.postgresql_connection()
with data_db(config) as db_conn:
perf = db_conn.get_performance(experiment_name=experiment_name)
return perf
def update_online_experiment(exp_combos, experiment_link):
"""Add a new hyperparameter combination to the experiment table."""
config = credentials.postgresql_connection()
with db(config) as db_conn:
db_conn.populate_db(exp_combos, experiment_link=experiment_link)