def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
# setup a logger
logger = setup_logger(config['logger'], name='experiment')
# we allow to set the random seed in the config file for reproducibility. However, when running on GPU, results
# will still be nondeterministic (due to nondeterministic behavior of tensorflow)
if 'random_seed' in config_global:
seed = config_global['random_seed']
logger.info('Using fixed random seed'.format(seed))
np.random.seed(seed)
tf.set_random_seed(seed)
with tf.Session(config=sess_config) as sess:
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
# We then wire together all the modules and start training
data = DataClass(config['data'], config_global, logger)
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
evaluation = EvaluationClass(config['evaluation'], config_global,
logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess, evaluation)
training.restore_best_epoch(model, data, sess, evaluation)
# perform evaluation, if required
if not config['evaluation'].get('skip', False):
logger.info('Evaluating')
evaluation.start(model, data, sess, valid_only=False)
else:
logger.info('Skipping evaluation')
logger.info('DONE')
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
logger = logging_utils.setup(config)
# Allow the gpu to be used in parallel
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
# We are now fetching all relevant modules. It is strictly required that these modules contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
# We then wire together all the modules and start training
data = DataClass(config['data'], config_global, logger)
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
evaluation = EvaluationClass(config['evaluation'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess, evaluation)
# perform evaluation, if required
if not config['evaluation'].get('skip', False):
logger.info('Evaluating')
evaluation.start(model, data, sess, valid_only=False)
else:
logger.info('Skipping evaluation')
logger.info('DONE')
def load_config_from_file(self):
'''
Load experiment configurations from file
'''
filename = tk.filedialog.askopenfilename(parent=self.root_view.root)
if not isinstance(filename, str) or filename == '': return
try:
config = load_config(filename)
except:
tk.messagebox.showinfo('Error',
'Invalid yaml file',
parent=self.root_view.root)
return
self.set_config(config)
def run_cmd(config_file, do_test):
config_eval = load_config(config_file)
config_experiment = load_config(config_eval['global']['experiment_config'])
def run():
"""This program is the starting point for evaluations without training"""
run_name = config_eval.get('run_name', 'undefined name')
config_eval_global = config_eval['global']
config_experiment_global = config_experiment['global']
output_dir = config_eval_global['evaluation_output']
restore_weights = config_eval_global.get('restore_epoch', 'best')
if os.path.exists(path.join(output_dir, 'log.txt')):
with open(path.join(output_dir, 'log.txt'), 'r') as f:
if 'Results for all datasets in dev' in f.read():
return 'exp. already finished'
if not os.path.exists(output_dir):
os.mkdir(output_dir)
write_config(config_eval, output_dir)
# setup a logger
logger = setup_logger(config_eval['logger'],
output_dir,
name='experiment')
# we allow to set the random seed in the config file for reproducibility. However, when running on GPU, results
# can still be nondeterministic
if 'random_seed' in config_experiment_global:
seed = config_experiment_global['random_seed']
logger.info('Using fixed random seed'.format(seed))
np.random.seed(seed)
tf.set_random_seed(seed)
multi_data_module_eval = MultiData(config_eval['data'],
config_eval_global, logger)
for dataset_config in config_eval['data']:
data_module = dataset_config['data-module']
splits = dataset_config.get('splits', ['train', 'dev', 'test'])
DataClass = importlib.import_module(data_module).component
data = DataClass(dataset_config, config_eval_global, logger)
multi_data_module_eval.add(data, splits)
model_module = config_experiment['model-module']
evaluation_module = config_eval['evaluation-module']
# The modules are now dynamically loaded
ModelClass = importlib.import_module(model_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
# We then wire together all the modules and start training
model = ModelClass(config_experiment['model'],
config_experiment_global, logger)
evaluation = EvaluationClass(config_eval['evaluation'],
config_eval_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
multi_data_module_eval.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.set_bert_configs_from_output_folder(
restore_weights) # sets necessary huggingface bert restore paths
model.build(multi_data_module_eval)
if hasattr(model.bert.config, 'adapter_attention'):
model.bert.bert.enable_adapters(unfreeze_adapters=True,
unfreeze_attention=True)
elif hasattr(model.bert.config, 'adapters'):
model.bert.bert.enable_adapters(unfreeze_adapters=True,
unfreeze_attention=False)
state = QATrainStatePyTorch(path.join(
config_experiment_global['output_path'], 'checkpoints'),
less_is_better=False,
logger=logger)
state.load(model.bert,
optimizer=None,
weights=restore_weights,
strict=False)
logger.info('Evaluating')
results = evaluation.start(model,
multi_data_module_eval,
valid_only=not do_test)
logger.info('DONE')
return {'run_name': run_name, 'results': results}
run()
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
# setup a logger
logger = logging.getLogger('experiment')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(config['logger']['level'])
handler_stdout.setFormatter(formatter)
logger.addHandler(handler_stdout)
if 'path' in config['logger']:
handler_file = logging.FileHandler(config['logger']['path'])
handler_file.setLevel(config['logger']['level'])
handler_file.setFormatter(formatter)
logger.addHandler(handler_file)
logger.setLevel(config['logger']['level'])
# Allow the gpu to be used in parallel
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
if 'max_threads' in config_global:
sess_config.intra_op_parallelism_threads = config_global['max_threads']
np.random.seed(1)
tf.set_random_seed(1)
with tf.Session(config=sess_config) as sess:
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(
evaluation_module).component if evaluation_module else None
# We then wire together all the modules and start training
data = DataClass(config['data'], config_global, logger)
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess)
# perform evaluation, if required
if EvaluationClass:
logger.info('Evaluating')
evaluation = EvaluationClass(config['evaluation'], config_global,
logger)
evaluation.start(model, data, sess)
else:
logger.info('No evaluation')
logger.info('DONE')
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
logger = logging_utils.setup(config)
# Allow the gpu to be used in parallel
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
# We are now fetching all relevant modules. It is strictly required that these modules contain a variable
# named 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
data = DataClass(config['data'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
results = defaultdict(lambda: list())
n_repetitions = config.get('repetitions', 1)
for i in range(n_repetitions):
tf.reset_default_graph()
logger.info('-' * 20)
logger.info('Repetition {}'.format(i))
with tf.Session(config=sess_config) as sess:
# We then wire together all the modules and start training
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
evaluation = EvaluationClass(config['evaluation'], config_global,
logger)
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess, evaluation)
# perform evaluation
logger.info('Evaluating')
result = evaluation.start(model, data, sess, valid_only=False)
for k, v in result.items():
results[k].append(v)
logger.info('DONE')
logger.info('Averaged results over {} runs:'.format(n_repetitions))
for key, values in results.items():
logger.info('AVG result over {} runs: {}={}'.format(
n_repetitions, key, np.mean(values)))
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
logger = logging_utils.setup(config)
data_writer = DataWriter(config['report'], logger)
data_writer.add_task('ArgMinPE')
data_module = config['data-module']
stl_model_module = config['stl-model-module']
stl_training_module = config['stl-training-module']
stl_evaluation_module = config.get('stl-evaluation-module', None)
mtl_model_module = config['mtl-model-module']
mtl_training_module = config['mtl-training-module']
mtl_evaluation_module = config.get('mtl-evaluation-module', None)
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
STLModelClass = importlib.import_module(stl_model_module).component
STLTrainingClass = importlib.import_module(stl_training_module).component
STLEvaluationClass = importlib.import_module(stl_evaluation_module).component
MTLModelClass = importlib.import_module(mtl_model_module).component
MTLTrainingClass = importlib.import_module(mtl_training_module).component
MTLEvaluationClass = importlib.import_module(mtl_evaluation_module).component
data = DataClass(config['data'], config_global, logger)
stl_evaluation = STLEvaluationClass(config['evaluation'], config_global, logger)
mtl_evaluation = MTLEvaluationClass(config['evaluation'], config_global, logger)
# setup the data (validate, create generators, load data, ...)
logger.info('Setting up the data')
data.setup()
results = OrderedDict()
for i in range(config['repetitions']):
tf.reset_default_graph()
logger.info('Repetition {}'.format(i))
with tf.Session(config=sess_config) as sess:
model = STLModelClass(config['model'], config_global, logger)
training = STLTrainingClass(config['training'], config_global, logger)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess, stl_evaluation)
result = stl_evaluation.start(model, data, sess, valid_only=False)
logger.info('Finished training the projection model with dev={}, test(en)={}, test(de)={}'.format(
result['valid'],
result['test-en'],
result['test-de']
))
if i == 0:
results['plain-transfer'] = {'dev': [], 'test-en': [], 'test-de': []}
results['plain-transfer']['dev'].append(result['valid'])
results['plain-transfer']['test-en'].append(result['test-en'])
results['plain-transfer']['test-de'].append(result['test-de'])
return
logger.info('Labeling the sentences of the parallel corpus...')
labeled_parallel_data_en, labeled_parallel_data_de = [], []
lbl_count_en, lbl_count_de = defaultdict(lambda: 0), defaultdict(lambda: 0)
for ((en, de),(en_vec, de_vec)) in zip(data.parallel_sentences, data.parallel_sentences_vectors):
p, = sess.run([model.predict], feed_dict={
model.input_sentence: [en_vec, de_vec],
model.dropout_keep_prob: 1.0
})
cls_en = data.classes[np.argmax(p[0])]
lbl_count_en[cls_en] += 1
labeled_parallel_data_en.append((en, de, p[0]))
cls_de = data.classes[np.argmax(p[1])]
lbl_count_de[cls_de] += 1
labeled_parallel_data_de.append((en, de, p[1]))
logger.info('Inferred labels (en): {}'.format(
' '.join(['{}={}'.format(k, v) for k, v in lbl_count_en.items()])
))
logger.info('Inferred labels (de): {}'.format(
' '.join(['{}={}'.format(k, v) for k, v in lbl_count_de.items()])
))
# MTL projection
for mode in [#'gold & projected_de', 'labeled_en & projected_de', 'gold+labeled_en & projected_de',
'gold & labeled_de',
# 'gold+projected_en & labeled_de',
#'gold+labeled_en+projected_en & projected_de+labeled_de',
'gold+labeled_en & labeled_de+projected_de']:
logger.info('Now training MTL, mode={}'.format(mode))
tf.reset_default_graph()
with tf.Session(config=sess_config) as sess:
merged_data = data.get_MTL(labeled_parallel_data_en, labeled_parallel_data_de, mode=mode)
model = MTLModelClass(config['model'], config_global, logger)
training = MTLTrainingClass(config['training'], config_global, logger)
logger.info('Building the model')
model.build(merged_data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, merged_data, sess, mtl_evaluation)
result = mtl_evaluation.start(model, merged_data, sess, valid_only=False)
logger.info(
'Finished training the MTL model (mode={}) with dev={}, test(en)={}, test(de)={}'.format(
mode,
result['valid'],
result['test-en'],
result['test-de']
))
if i == 0:
results['mtl-{}'.format(mode)] = {'dev': [], 'test-en': [], 'test-de': []}
results['mtl-{}'.format(mode)]['dev'].append(result['valid'])
results['mtl-{}'.format(mode)]['test-en'].append(result['test-en'])
results['mtl-{}'.format(mode)]['test-de'].append(result['test-de'])
for mode in ['projected_de', 'gold+projected_de', 'labeled_en+projected_de', 'gold+labeled_en+projected_de',
#'labeled_de',
#'gold+labeled_de',
# 'gold+projected_en+labeled_de', 'gold+projected_en+labeled_en+labeled_de+projected_de'
#'gold+labeled_en+labeled_de+projected_de'
]:
logger.info('Now training with merged data, mode={}'.format(mode))
tf.reset_default_graph()
with tf.Session(config=sess_config) as sess:
merged_data = data.get_merge(labeled_parallel_data_en, labeled_parallel_data_de, mode=mode)
model = STLModelClass(config['model'], config_global, logger)
training = STLTrainingClass(config['training'], config_global, logger)
logger.info('Building the model')
model.build(merged_data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, merged_data, sess, stl_evaluation)
result = stl_evaluation.start(model, merged_data, sess, valid_only=False, mode=mode)
logger.info(
'Finished training the merged model (mode={}) with dev={}, test(en)={}, test(de)={}'.format(
mode,
result['valid'],
result['test-en'],
result['test-de']
))
if i == 0:
results['merge-{}'.format(mode)] = {'dev': [], 'test-en': [], 'test-de': []}
results['merge-{}'.format(mode)]['dev'].append(result['valid'])
results['merge-{}'.format(mode)]['test-en'].append(result['test-en'])
results['merge-{}'.format(mode)]['test-de'].append(result['test-de'])
logger.debug(json.dumps(results))
data_writer.add_data_all(results.items())
data_writer.finish()
logger.info('DONE')
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
# setup a logger
logger = logging.getLogger('experiment')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(config['logger']['level'])
handler_stdout.setFormatter(formatter)
logger.addHandler(handler_stdout)
if 'path' in config['logger']:
handler_file = logging.FileHandler(config['logger']['path'])
handler_file.setLevel(config['logger']['level'])
handler_file.setFormatter(formatter)
logger.addHandler(handler_file)
logger.setLevel(config['logger']['level'])
# Allow the gpu to be used in parallel
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
if 'max_threads' in config_global:
sess_config.intra_op_parallelism_threads = config_global['max_threads']
np.random.seed(1)
tf.set_random_seed(1)
with tf.Session(config=sess_config) as sess:
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component if evaluation_module else None
# We then wire together all the modules and start training
data = DataClass(config['data'], config_global, logger)
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess)
# perform evaluation, if required
if EvaluationClass:
logger.info('Evaluating')
evaluation = EvaluationClass(config['evaluation'], config_global, logger)
evaluation.start(model, data, sess)
else:
logger.info('No evaluation')
logger.info('DONE')
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
logger = logging_utils.setup(config)
data_writer = DataWriter(config['report'], logger)
try:
for task in config['tasks']:
task_name = task['name']
repetitions = task['repetitions']
with open(task['config'], 'r') as f:
task_config = f.read()
data_writer.add_task(task_name)
logger.info('Task {}'.format(task_name))
results = []
for run in task['runs']:
run_config = read_config(task_config.format(**run))
run_config_global = run_config['global']
run_name = run_config.get('name', json.dumps(run))
logger.info('Run {}'.format(run_name))
data_module = run_config['data-module']
model_module = run_config['model-module']
training_module = run_config['training-module']
evaluation_module = run_config.get('evaluation-module')
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
data = DataClass(run_config['data'], run_config_global, logger)
logger.debug('Setting up the data')
data.setup()
best_lr = 0
best_dev_score = 0
best_dev_results = []
for lr in [0.05, 0.01, 0.005, 0.001, 0.0005]:
train_conf = dict(run_config['training'])
train_conf['initial_learning_rate'] = lr
run_results = OrderedDict()
for i in range(repetitions):
logger.info('Repetition {}'.format(i))
data.reshuffle(i) # e.g. random subsample
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
tf.reset_default_graph()
with TemporaryDirectory() as tmp_dir:
train_conf['save_folder'] = tmp_dir
with tf.Session(config=sess_config) as sess:
model = ModelClass(run_config['model'], run_config_global, logger)
training = TrainingClass(train_conf, run_config_global, logger)
evaluation = EvaluationClass(run_config['evaluation'], run_config_global, logger)
# build the model (e.g. compile it)
logger.debug('Building the model')
model.build(data, sess)
# start the training process
logger.debug('Starting the training process')
training.start(model, data, sess, evaluation)
logger.debug('Evaluating')
result = evaluation.start(model, data, sess, valid_only=False)
logger.info('Got results for {}: {}'.format(run_name, json.dumps(result)))
for key, value in result.items():
if key not in run_results:
run_results[key] = []
run_results[key].append(value)
aggregated_dev = np.mean(run_results['valid'])
if aggregated_dev > best_dev_score:
best_dev_score = aggregated_dev
best_dev_results = run_results
best_lr = lr
logger.info('best result {} for run with learning rate {}'.format(best_dev_score, best_lr))
results.append((run_name, best_dev_results))
data_writer.add_data_all(results)
finally:
data_writer.finish()
logger.info('DONE')
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
config_random_search = config['random_search']
logger = setup_logger(config['logger'], 'random_search')
parameter_value_sampling = ParameterValueSampling(config_random_search['parameters'])
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
logger.info('Setting up the data')
data = DataClass(config['data'], config_global, logger)
data.setup()
logger.info('Now starting the random search')
stop_time = datetime.datetime.now() + datetime.timedelta(hours=config_random_search['max_hours'])
logger.info('Stopping at {}'.format(stop_time))
if not os.path.exists(config_random_search['output_path']):
os.mkdir(config_random_search['output_path'])
n_runs = 0
best_run_id, best_run_score, best_run_params = 0, 0, None
while datetime.datetime.now() < stop_time:
n_runs += 1
run_path = os.path.join(config_random_search['output_path'], 'run-{}'.format(n_runs))
# we build a new config according to the random search configuration
random_params = parameter_value_sampling.get_random_values()
logger.info('-' * 20)
logger.info('Run {}'.format(n_runs))
logger.info('With params: {}'.format(json.dumps(random_params)))
config_run = replace_dict_values(config, random_params + [('training.remove_save_folder_on_start', True)])
with open('{}.yaml'.format(run_path), 'w') as f:
yaml.dump(config_run, f, default_flow_style=False)
# we add a new handler to the logger to print all messages to a separate file
run_log_handler = logging.FileHandler('{}.log'.format(run_path))
run_log_handler.setLevel(config['logger']['level'])
run_log_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(run_log_handler)
tf.reset_default_graph()
with tf.Session(config=sess_config) as sess:
model = ModelClass(config_run['model'], config_global, logger)
training = TrainingClass(config_run['training'], config_global, logger)
evaluation = EvaluationClass(config_run['evaluation'], config_global, logger)
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
best_epoch, best_score = training.start(model, data, sess, evaluation)
logger.info('DONE')
# We write the dev score to a file
with open('{}.score'.format(run_path), 'w') as f:
f.write('Score: {}\n'.format(best_score))
f.write('In epoch: {}\n'.format(best_epoch))
f.write('With params: {}\n'.format(json.dumps(random_params)))
if best_run_score < best_score:
best_run_id = n_runs
best_run_score = best_score
best_run_params = random_params
logger.removeHandler(run_log_handler)
logger.info('-' * 20)
logger.info('Now stopping. Did perform {} runs.'.format(n_runs))
logger.info('Best run: id={} with score {}'.format(best_run_id, best_run_score))
logger.info('Parameters={}'.format(json.dumps(best_run_params)))
def run(config_file):
"""This program allows to perform hyperparameter optimization with grid search and cross validation.
"""
config = load_config(config_file)
# to reproduce the results
np.random.seed(1)
tf.set_random_seed(1)
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment, experiment.Trainer
# or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
# setup a logger
logger = logging.getLogger('experiment')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(config['logger']['level'])
handler_stdout.setFormatter(formatter)
logger.addHandler(handler_stdout)
if 'path' in config['logger']:
handler_file = logging.FileHandler(config['logger']['path'])
handler_file.setLevel(config['logger']['level'])
handler_file.setFormatter(formatter)
logger.addHandler(handler_file)
logger.setLevel(config['logger']['level'])
# We then wire together all the modules
config_global = config['global']
config_optimization = config['optimization']
logger.info('Setting up the data')
data_complete = DataClass(config['data'], config_global, logger)
data_complete.setup()
# Cross fold optimization
# We first need to create all the configuration choices for grid search
optimization_parameters = config_optimization['parameters']
grid = list(itertools.product(*config_optimization['parameters'].values()))
logger.info('We have {} different hyperparameter combinations'.format(
len(grid)))
# We now go over all choices and perform cross-fold validation
avg_scores = []
for configuration_values_overwrite in grid:
parameter_choices = list(
zip(optimization_parameters.keys(),
configuration_values_overwrite))
config_run = replace_dict_values(config, parameter_choices)
logger.info('-' * 40)
logger.info('Checking configuration {}'.format(
json.dumps(parameter_choices)))
# Run each fold
n_folds = config_optimization['folds']
scores = []
for fold_i in range(n_folds):
logger.info('Starting fold {}/{}'.format(fold_i + 1, n_folds))
with tf.Session(config=sess_config) as sess:
training = TrainingClass(config_run['training'], config_global,
logger)
model = ModelClass(config_run['model'], config_global, logger)
model.build(data_complete, sess)
data_fold = data_complete.get_fold_data(fold_i, n_folds)
best_epoch, best_score = training.start(model, data_fold, sess)
scores.append(best_score)
logger.info('Fold {}/{} performance: {}'.format(
fold_i + 1, n_folds, best_score))
logger.info('-' * 20)
training.remove_checkpoints()
tf.reset_default_graph()
logger.info('Training ended')
logger.info('All scores: {}'.format(json.dumps(scores)))
avg_score = np.mean(scores)
logger.info(
'Avg score for current configuration: {}'.format(avg_score))
avg_scores.append(avg_score)
best_configuration = grid[np.argmax(avg_scores)]
logger.info('Grid search completed')
logger.info('Best configuration: {} (score={})'.format(
best_configuration, max(avg_scores)))
logger.info('-')
logger.info('All configurations: {}'.format(json.dumps(grid)))
logger.info('All scores: {}'.format(avg_scores))
logger.info('DONE')
def run_cmd(config_file):
config = load_config(config_file)
def run():
"""This program is the starting point for every experiment. It pulls together the configuration and all
necessary experiment classes to load
"""
run_name = config.get('run_name', 'undefined name')
config_global = config['global']
if not os.path.exists(config_global['output_path']):
os.mkdir(config_global['output_path'])
write_config(config, config_global['output_path'])
# setup a logger
logger = setup_logger(config['logger'], config_global['output_path'], name='experiment')
# we allow to set the random seed in the config file for reproducibility. However, when running on GPU, results
# can still be nondeterministic
if 'random_seed' in config_global:
seed = config_global['random_seed']
logger.info('Using fixed random seed'.format(seed))
np.random.seed(seed)
tf.set_random_seed(seed)
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
multi_data_module = MultiData(config['data'], config, logger)
for dataset_config in config['data']:
data_module = dataset_config['data-module']
splits = dataset_config.get('splits', ['train', 'dev', 'test'])
DataClass = importlib.import_module(data_module).component
data = DataClass(dataset_config, config_global, logger)
multi_data_module.add(data, splits)
model_module = config['model-module']
training_module = config['training-module']
evaluation_module = config.get('evaluation-module', None)
# The modules are now dynamically loaded
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
EvaluationClass = importlib.import_module(evaluation_module).component
if "adapter_mean_estimation-module" in config:
ame_module = config["adapter_mean_estimation-module"]
AMEClass = importlib.import_module(ame_module).component
ame = AMEClass(config['adapter_mean_estimation'], config_global, logger)
# We then wire together all the modules and start training
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
evaluation = EvaluationClass(config['evaluation'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
multi_data_module.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(multi_data_module)
# start the training process
if not config["training"].get("skip", False):
logger.info('Starting the training process')
training.start(model, multi_data_module, evaluation)
dev_score = training.restore_best_epoch(model, multi_data_module, evaluation)
# perform evaluation, if required
if not config['evaluation'].get('skip', False):
logger.info('Evaluating')
results = evaluation.start(model, multi_data_module, valid_only=False)
else:
logger.info('Skipping evaluation')
results = {'dev': dev_score}
logger.info('DONE')
return {
'run_name': run_name,
'results': results
}
run()
def run(config_file, port):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
global sess, model, config, data
config = load_config(config_file)
config_global = config['global']
# setup a logger
logger = logging.getLogger('experiment')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(config['logger']['level'])
handler_stdout.setFormatter(formatter)
logger.addHandler(handler_stdout)
if 'path' in config['logger']:
handler_file = logging.FileHandler(config['logger']['path'])
handler_file.setLevel(config['logger']['level'])
handler_file.setFormatter(formatter)
logger.addHandler(handler_file)
logger.setLevel(config['logger']['level'])
# Allow the gpu to be used in parallel
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
if 'max_threads' in config_global:
sess_config.intra_op_parallelism_threads = config_global['max_threads']
# we allow to set the random seed in the config file for reproducibility. However, when running on GPU, results
# will still be nondeterministic (due to nondeterministic behavior of tensorflow)
if 'random_seed' in config_global:
seed = config_global['random_seed']
logger.info('Using fixed random seed'.format(seed))
np.random.seed(seed)
tf.set_random_seed(seed)
sess = tf.InteractiveSession(config=sess_config)
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
# We then wire together all the modules and start training
data = DataClass(config['data'], config_global, logger)
model = ModelClass(config['model'], config_global, logger)
training = TrainingClass(config['training'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# start the training process
logger.info('Starting the training process')
training.start(model, data, sess)
app.run(debug=False, port=port, host="0.0.0.0")
sess.close()
def run(config_file):
"""This program is the starting point for every experiment. It pulls together the configuration and all necessary
experiment classes to load
"""
config = load_config(config_file)
config_global = config['global']
# setup a logger
logger = setup_logger(config['logger'], name='service')
# we allow to set the random seed in the config file for reproducibility. However, when running on GPU, results
# will still be nondeterministic (due to nondeterministic behavior of tensorflow)
if 'random_seed' in config_global:
seed = config_global['random_seed']
logger.info('Using fixed random seed'.format(seed))
np.random.seed(seed)
tf.set_random_seed(seed)
with tf.Session(config=sess_config) as sess:
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment,
# experiment.Trainer or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
# We then wire together all the modules and start training
data = DataClass(config['data'], config_global, logger)
model = ModelClass(config['model'], config_global, logger)
# setup the data (validate, create generators, load data, or else)
logger.info('Setting up the data')
data.setup()
# build the model (e.g. compile it)
logger.info('Building the model')
model.build(data, sess)
# restore the model
logger.info('Restoring the model')
saver = tf.train.Saver()
saver.restore(
sess,
tf.train.latest_checkpoint(config['training']['save_folder']))
logger.info('Restored the model')
# serve service
while True:
line = sys.stdin.readline().strip()
if line == 'START QUERY':
question = sys.stdin.readline()
q_tokens = question.strip().split()
question = convert_input(q_tokens, data.vocab_to_index)
n_answer = int(sys.stdin.readline().strip())
answers = []
for _ in range(n_answer):
answer = sys.stdin.readline()
a_tokens = answer.strip().split()
answer = convert_input(a_tokens, data.vocab_to_index)
answers.append(answer)
question = np.array([
data.get_item_vector(question, 150)
for _ in range(len(answers))
])
answers = np.array(
[data.get_item_vector(answer, 300) for answer in answers])
# run model
scores, = sess.run(
[model.predict],
feed_dict={
model.input_question: question,
model.input_answer: answers,
model.dropout_keep_prob: 1.0,
})
selected = np.argsort(-scores)[:5]
enablePrint()
sys.stdout.write(
(' '.join([str(idx) for idx in selected]) + '\n'))
sys.stdout.flush()
blockPrint()
def run(config_file):
"""This program allows to perform hyperparameter optimization with grid search and cross validation.
"""
config = load_config(config_file)
# to reproduce the results
np.random.seed(1)
tf.set_random_seed(1)
# We are now fetching all relevant modules. It is strictly required that these module contain a variable named
# 'component' that points to a class which inherits from experiment.Data, experiment.Experiment, experiment.Trainer
# or experiment.Evaluator
data_module = config['data-module']
model_module = config['model-module']
training_module = config['training-module']
# The modules are now dynamically loaded
DataClass = importlib.import_module(data_module).component
ModelClass = importlib.import_module(model_module).component
TrainingClass = importlib.import_module(training_module).component
# setup a logger
logger = logging.getLogger('experiment')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler_stdout = logging.StreamHandler(sys.stdout)
handler_stdout.setLevel(config['logger']['level'])
handler_stdout.setFormatter(formatter)
logger.addHandler(handler_stdout)
if 'path' in config['logger']:
handler_file = logging.FileHandler(config['logger']['path'])
handler_file.setLevel(config['logger']['level'])
handler_file.setFormatter(formatter)
logger.addHandler(handler_file)
logger.setLevel(config['logger']['level'])
# We then wire together all the modules
config_global = config['global']
config_optimization = config['optimization']
logger.info('Setting up the data')
data_complete = DataClass(config['data'], config_global, logger)
data_complete.setup()
# Cross fold optimization
# We first need to create all the configuration choices for grid search
optimization_parameters = config_optimization['parameters']
grid = list(itertools.product(*config_optimization['parameters'].values()))
logger.info('We have {} different hyperparameter combinations'.format(len(grid)))
# We now go over all choices and perform cross-fold validation
avg_scores = []
for configuration_values_overwrite in grid:
parameter_choices = list(zip(optimization_parameters.keys(), configuration_values_overwrite))
config_run = replace_dict_values(config, parameter_choices)
logger.info('-' * 40)
logger.info('Checking configuration {}'.format(json.dumps(parameter_choices)))
# Run each fold
n_folds = config_optimization['folds']
scores = []
for fold_i in range(n_folds):
logger.info('Starting fold {}/{}'.format(fold_i + 1, n_folds))
with tf.Session(config=sess_config) as sess:
training = TrainingClass(config_run['training'], config_global, logger)
model = ModelClass(config_run['model'], config_global, logger)
model.build(data_complete, sess)
data_fold = data_complete.get_fold_data(fold_i, n_folds)
best_epoch, best_score = training.start(model, data_fold, sess)
scores.append(best_score)
logger.info('Fold {}/{} performance: {}'.format(fold_i + 1, n_folds, best_score))
logger.info('-' * 20)
training.remove_checkpoints()
tf.reset_default_graph()
logger.info('Training ended')
logger.info('All scores: {}'.format(json.dumps(scores)))
avg_score = np.mean(scores)
logger.info('Avg score for current configuration: {}'.format(avg_score))
avg_scores.append(avg_score)
best_configuration = grid[np.argmax(avg_scores)]
logger.info('Grid search completed')
logger.info('Best configuration: {} (score={})'.format(best_configuration, max(avg_scores)))
logger.info('-')
logger.info('All configurations: {}'.format(json.dumps(grid)))
logger.info('All scores: {}'.format(avg_scores))
logger.info('DONE')