def train_tasks(parameters):
# GPU setting
if parameters['use_cuda']:
os.environ['CUDA_VISIBLE_DEVICES'] = parameters['gpu']
if len(parameters['tasks']) > 1:
print('please train 1 task at once')
return
task_id = parameters['tasks'][0]
# Log setting
log_suffix = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
if parameters['note']: log_suffix = parameters['note'] + '_' + log_suffix
log_file = log_dir + 'split_train_task_{}_{}.txt'.format(task_id, log_suffix)
train_logger = logger(log_file, parameters)
train_logger.log('\n')
parameters['logger'] = train_logger
# start training
task = tasks[task_id]
data_prefix = 'task_' + str(task_id)
parameters['logger'].log_dict(task)
parameters['logger'].log('dataset prefix: ' + data_prefix)
parameters['model_prefix'] = 'split_train_task_{}'.format(task_id)
# Dataset
X_train = np.load(saves_directory + data_prefix + '_X_train.npy', allow_pickle=True)
y_train = np.load(saves_directory + data_prefix + '_y_train.npy', allow_pickle=True)
X_val = np.load(saves_directory + data_prefix + '_X_val.npy', allow_pickle=True)
y_val = np.load(saves_directory + data_prefix + '_y_val.npy', allow_pickle=True)
X_test = np.load(saves_directory + data_prefix + '_X_test.npy', allow_pickle=True)
y_test = np.load(saves_directory + data_prefix + '_y_test.npy', allow_pickle=True)
dataset_indices = [i+1 for i in paper_tasks.dataset_indices[feature_set]]
field_indices = [i+2 for i in paper_tasks.field_indices[feature_set]]
if task['dataset'] == 'dataset':
X_train = X_train [:, dataset_indices]
X_val = X_val [:, dataset_indices]
X_test = X_test [:, dataset_indices]
else:
assert task['dataset'] == 'field'
X_train = X_train [:, field_indices]
X_val = X_val [:, field_indices]
X_test = X_test [:, field_indices]
parameters['logger'].log('loaded train X size: ' + str(X_train.shape))
parameters['logger'].log('loaded train y size: ' + str(y_train.shape))
parameters['logger'].log('loaded val X size: ' + str(X_val.shape))
parameters['logger'].log('loaded val y size: ' + str(y_val.shape))
parameters['logger'].log('loaded test X size: ' + str(X_test.shape))
parameters['logger'].log('loaded test y size: ' + str(y_test.shape))
# get dataloader
train_dataloader, val_dataloader, test_dataloader = train.load_datasets(
X_train, y_train, X_val, y_val, parameters, X_test, y_test)
train.train(
train_dataloader,
val_dataloader,
test_dataloader,
parameters,
models_directory=models_directory)
def main():
# all the parameters for the script can be adjusted here
# num_epochs: the max number of epochs we will train the NN for
# hidden_sizes: the number of neurons in each hidden layer, enter it as a list
# output_dim: the dimension of the output. Since outputs are 'line', 'scatter', 'bar', it's 3
# weight_decay: how much to decay LR in the NN. This can be set to 0 since we decrease LR already through
# the ReduceLROnPlateau() function
# dropout: the dropout in each layer
# patience: how many epochs we go through (with a near constant learning rate, this threshold is adjusted using
# threshold) before dropping learning rate by a factor of 10
# model_prefix: all models will be loaded/saved with the prefix of the file in the beginning
# save_model: save each epoch's model onto models/ folder.
# print_test: print test accuracies into test.txt
# test_best: test the test accuracy of the best model we've found (best
# model determined using val accuracy)
# note: training is automatically stopped when learning rate < 0.01 *
# starting learning rate
parameters = {
'batch_size': 200,
'num_epochs': 100,
'hidden_sizes': [800, 800, 800],
'learning_rate': 5e-4,
'output_dim': 3,
'weight_decay': 0,
'dropout': 0.00,
'patience': 20,
'threshold': 1e-3,
'model_prefix': 'agg',
'save_model': False,
'print_test': True,
'test_best': False
}
# LOAD loads the unfiltered features from the .csv files and converts them into filtered .npy files into ~/saves
# TRAIN trains using the given parameters and .npy files
# EVAL evaluates prefix_.model_number (giving you test accuracy)
assert len(
sys.argv) >= 2, 'You must specify a command LOAD, TRAIN, or EVAL'
assert (parameters['model_prefix']
), 'You must specify a prefix for the model name'
if parameters['test_best']:
assert parameters[
'save_model'], 'You must save a model to test the best version!'
command = sys.argv[1].lower()
if command == 'load':
X, y = load_features()
# here, we split 10% of examples into val, and 10% into test
util.save_matrices_to_disk(X, y, [0.1, 0.1], saves_directory,
parameters['model_prefix'], num_datapoints)
return
X_train, y_train, X_val, y_val, X_test, y_test = util.load_matrices_from_disk(
saves_directory, parameters['model_prefix'], num_datapoints)
if command == 'train':
train_dataloader, val_dataloader, test_dataloader = train.load_datasets(
X_train,
y_train,
X_val,
y_val,
parameters,
X_test=X_test,
y_test=y_test)
train.train(train_dataloader, val_dataloader, test_dataloader,
parameters)
elif command == 'eval':
assert len(sys.argv) >= 3
model_suffix = sys.argv[2]
evaluate.evaluate(model_suffix, X_test, y_test, parameters)
else:
assert False, 'The command must either be LOAD, TRAIN, or EVAL'
def main():
tasks = [None,
{'outcome_variable_name': 'all_one_trace_type',
'prediction_task': 'two',
'sampling_mode': 'over',
'pref_id': 1,
'dataset': 'dataset'},
{'outcome_variable_name': 'all_one_trace_type',
'prediction_task': 'three',
'sampling_mode': 'over',
'pref_id': 2,
'dataset': 'dataset'},
{'outcome_variable_name': 'all_one_trace_type', 'prediction_task': 'six',
'sampling_mode': 'over', 'pref_id': 3, 'dataset': 'dataset'},
{'outcome_variable_name': 'has_single_src', 'prediction_task': 'two',
'sampling_mode': 'over', 'pref_id': 4, 'dataset': 'dataset'},
{'outcome_variable_name': 'num_x_axes', 'prediction_task': 'numeric',
'sampling_mode': 100, 'pref_id': 5, 'dataset': 'dataset'}, #10000
{'outcome_variable_name': 'num_y_axes', 'prediction_task': 'numeric',
'sampling_mode': 100, 'pref_id': 6, 'dataset': 'dataset'}, #10000
{'outcome_variable_name': 'trace_type', 'prediction_task': 'two',
'sampling_mode': 'over', 'pref_id': 7, 'dataset': 'field'},
{'outcome_variable_name': 'trace_type', 'prediction_task': 'three',
'sampling_mode': 'over', 'pref_id': 8, 'dataset': 'field'},
{'outcome_variable_name': 'trace_type', 'prediction_task': 'six',
'sampling_mode': 'over', 'pref_id': 9, 'dataset': 'field'},
{'outcome_variable_name': 'is_single_src', 'prediction_task': 'two',
'sampling_mode': 'over', 'pref_id': 10, 'dataset': 'field'},
{'outcome_variable_name': 'is_x_or_y', 'prediction_task': 'two',
'sampling_mode': 'over', 'pref_id': 11, 'dataset': 'field'},
]
for i in [1]: #range(2, 12): # range(7, len(tasks)):
task = tasks[i]
model_prefix = 'paper_' + task['dataset'] + '_' + str(task['pref_id'])
parameters = {
'batch_size': 200,
'num_epochs': 100,
'hidden_sizes': [1000, 1000, 1000],
'learning_rate': 5e-4,
'weight_decay': 0,
'dropout': 0.00,
'patience': 10,
'threshold': 1e-3,
'model_prefix': model_prefix,
# uncomment this if you want to print test accuracies/save model
'only_train': False,
'save_model': True,
'print_test': True,
# for constructing learning curves
'dataset_ratios': [0.01, 0.1, 0.5, 1.0],
'test_best': True,
'use_cuda': False
}
if parameters['use_cuda'] == True:
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
for feature_set in [3]: #[0, 1, 2, 3]: # range(0, 4): # dimensions, types, values, names
assert len(sys.argv) >= 2, 'You must specify a command LOAD, TRAIN, or EVAL'
assert(parameters['model_prefix']), 'You must specify a prefix for the model name'
if 'test_best' in parameters and parameters['test_best']:
assert parameters['save_model'], 'You must save a model to test the best version!'
command = sys.argv[1].lower()
if command == 'load':
X, y = load_features(task)
util.save_matrices_to_disk(
X, y, [0.2, 0.2], saves_directory, parameters['model_prefix'], num_datapoints)
else:
X_train, y_train, X_val, y_val, X_test, y_test = util.load_matrices_from_disk(
saves_directory, parameters['model_prefix'], num_datapoints)
if task['dataset'] == 'dataset':
X_train = X_train[:, dataset_indices[feature_set]]
X_val = X_val[:, dataset_indices[feature_set]]
X_test = X_test[:, dataset_indices[feature_set]]
else:
assert task['dataset'] == 'field'
X_train = X_train[:, field_indices[feature_set]]
X_val = X_val[:, field_indices[feature_set]]
X_test = X_test[:, field_indices[feature_set]]
print('loaded dimensions are', X_train.shape)
print('task_num and feature_set:(' +
str(task['pref_id']) + ',' + str(feature_set) + ')')
if command == 'train':
log_file = log_dir + 'training_task_' + str(task['pref_id']) + '.txt'
train_logger = logger(log_file, task)
train_dataloader, val_dataloader, test_dataloader = train.load_datasets(
X_train, y_train, X_val, y_val, parameters, X_test, y_test, train_logger)
train.train(
train_dataloader,
val_dataloader,
test_dataloader,
parameters,
models_directory=models_directory,
suffix=suffix,
logger=train_logger)
elif command == 'eval':
assert len(sys.argv) >= 3
model_suffix = sys.argv[2]
log_file = log_dir + 'testing_task_' + str(task['pref_id']) + '.txt'
test_logger = logger(log_file, task)
train_dataloader, val_dataloader, test_dataloader = train.load_datasets(
X_train, y_train, X_val, y_val, parameters, X_test, y_test, test_logger)
evaluate.evaluate(
model_suffix, test_dataloader, parameters, models_directory)
else:
assert False, 'The command must either be LOAD, TRAIN, or EVAL'