def evaluate(self,
dataset: Dataset,
write_out: bool = False,
log_progress: int = 0,
name: str = None) -> Dict[str, Any]:
"""Run the model on a given dataset and evaluate the outputs.
Args:
dataset: The dataset on which the model will be executed.
write_out: Flag whether the outputs should be printed to a file
defined in the dataset object.
log_progress: log progress every X seconds
name: The name of the evaluated dataset
Returns:
Dictionary of evaluation names and their values which includes the
metrics applied on respective series loss and loss values from the
run.
"""
execution_results, output_data, f_dataset = self.run_model(
dataset, write_out, log_progress)
evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e
for e in self.model.evaluation]
with self.graph.as_default():
eval_result = evaluation(evaluators, f_dataset, execution_results,
output_data)
if eval_result:
print_final_evaluation(eval_result, name)
return eval_result
def main() -> None:
# pylint: disable=no-member,broad-except
if len(sys.argv) != 3:
print("Usage: run.py <run_ini_file> <test_datasets>")
exit(1)
test_datasets = Configuration()
test_datasets.add_argument('test_datasets')
test_datasets.add_argument('variables')
CONFIG.load_file(sys.argv[1])
CONFIG.build_model()
test_datasets.load_file(sys.argv[2])
test_datasets.build_model()
datesets_model = test_datasets.model
initialize_for_running(CONFIG.model.output, CONFIG.model.tf_manager,
datesets_model.variables)
print("")
evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e
for e in CONFIG.model.evaluation]
for dataset in datesets_model.test_datasets:
execution_results, output_data = run_on_dataset(
CONFIG.model.tf_manager,
CONFIG.model.runners,
dataset,
CONFIG.model.postprocess,
write_out=True)
# TODO what if there is no ground truth
eval_result = evaluation(evaluators, dataset, CONFIG.model.runners,
execution_results, output_data)
if eval_result:
print_final_evaluation(dataset.name, eval_result)
def evaluate(self,
dataset: Dataset,
write_out: bool = False,
log_progress: int = 0,
name: str = None) -> Dict[str, Any]:
"""Run the model on a given dataset and evaluate the outputs.
Args:
dataset: The dataset on which the model will be executed.
write_out: Flag whether the outputs should be printed to a file
defined in the dataset object.
log_progress: log progress every X seconds
name: The name of the evaluated dataset
Returns:
Dictionary of evaluation names and their values which includes the
metrics applied on respective series loss and loss values from the
run.
"""
execution_results, output_data, f_dataset = self.run_model(
dataset, write_out, log_progress)
evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e
for e in self.model.evaluation]
with self.graph.as_default():
eval_result = evaluation(
evaluators, f_dataset, execution_results, output_data)
if eval_result:
print_final_evaluation(eval_result, name)
return eval_result
def main() -> None:
# pylint: disable=no-member,broad-except
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("config",
metavar="INI-FILE",
help="the configuration file of the experiment")
parser.add_argument('datasets',
metavar='INI-TEST-DATASETS',
help="the configuration of the test datasets")
parser.add_argument("-g",
"--grid",
dest="grid",
action="store_true",
help="look at the SGE variables for slicing the data")
args = parser.parse_args()
test_datasets = Configuration()
test_datasets.add_argument('test_datasets')
test_datasets.add_argument('variables')
CONFIG.load_file(args.config)
CONFIG.build_model()
test_datasets.load_file(args.datasets)
test_datasets.build_model()
datasets_model = test_datasets.model
initialize_for_running(CONFIG.model.output, CONFIG.model.tf_manager,
datasets_model.variables)
print("")
evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e
for e in CONFIG.model.evaluation]
if args.grid and len(datasets_model.test_datasets) > 1:
raise ValueError("Only one test dataset supported when using --grid")
for dataset in datasets_model.test_datasets:
if args.grid:
if ("SGE_TASK_FIRST" not in os.environ
or "SGE_TASK_LAST" not in os.environ
or "SGE_TASK_STEPSIZE" not in os.environ
or "SGE_TASK_ID" not in os.environ):
raise EnvironmentError(
"Some SGE environment variables are missing")
length = int(os.environ["SGE_TASK_STEPSIZE"])
start = int(os.environ["SGE_TASK_ID"]) - 1
end = int(os.environ["SGE_TASK_LAST"]) - 1
if start + length > end:
length = end - start + 1
log("Running grid task {} starting at {} with step {}".format(
start // length, start, length))
dataset = dataset.subset(start, length)
if CONFIG.model.runners_batch_size is None:
runners_batch_size = CONFIG.model.batch_size
else:
runners_batch_size = CONFIG.model.runners_batch_size
execution_results, output_data = run_on_dataset(
CONFIG.model.tf_manager,
CONFIG.model.runners,
dataset,
CONFIG.model.postprocess,
write_out=True,
batch_size=runners_batch_size,
log_progress=60)
# TODO what if there is no ground truth
eval_result = evaluation(evaluators, dataset, CONFIG.model.runners,
execution_results, output_data)
if eval_result:
print_final_evaluation(dataset.name, eval_result)