def run_model( self, dataset: Dataset, write_out: bool = False, log_progress: int = 0 ) -> Tuple[List[ExecutionResult], Dict[str, List], Dict[str, List]]: """Run the model on a given dataset. Args: dataset: The dataset on which the model will be executed. write_out: Flag whether the outputs should be printed to a file defined in the dataset object. log_progress: log progress every X seconds Returns: A list of `ExecutionResult`s and a dictionary of the output series. """ if not self._model_built: self.build_model() if not self._vars_loaded: self.load_variables() with self.graph.as_default(): return run_on_dataset(self.model.tf_manager, self.model.runners, self.model.dataset_runner, dataset, self.model.postprocess, write_out=write_out, log_progress=log_progress)
def main(): # pylint: disable=no-member,broad-except if len(sys.argv) != 3: print("Usage: run.py <run_ini_file> <test_datasets>") exit(1) test_datasets = Configuration() test_datasets.add_argument('test_datasets') args, sess = initialize_for_running(sys.argv[1]) datasets_args = test_datasets.load_file(sys.argv[2]) print("") try: for dataset in datasets_args.test_datasets: check_dataset_and_coders(dataset, args.encoders) except Exception as exc: log(exc.message, color='red') exit(1) for dataset in datasets_args.test_datasets: _, _, evaluation = run_on_dataset( sess, args.runner, args.encoders + [args.decoder], args.decoder, dataset, args.evaluation, args.postprocess, write_out=True) if evaluation: print_dataset_evaluation(dataset.name, evaluation)
def post_request(): start_time = datetime.datetime.now() request_data = request.get_json() if request_data is None: response_data = {"error": "No data were provided."} code = 400 else: args = APP.config['args'] try: dataset = Dataset("request", request_data, {}) # TODO check the dataset # check_dataset_and_coders(dataset, args.encoders) _, response_data = run_on_dataset( args.tf_manager, args.runners, dataset, args.postprocess, write_out=False) code = 200 # pylint: disable=broad-except except Exception as exc: response_data = {'error': str(exc)} code = 400 response_data['duration'] = ( datetime.datetime.now() - start_time).total_seconds() json_response = json.dumps(response_data) response = flask.Response(json_response, content_type='application/json; charset=utf-8') response.headers.add('content-length', len(json_response.encode('utf-8'))) response.status_code = code return response
def main() -> None: # pylint: disable=no-member,broad-except if len(sys.argv) != 3: print("Usage: run.py <run_ini_file> <test_datasets>") exit(1) test_datasets = Configuration() test_datasets.add_argument('test_datasets') test_datasets.add_argument('variables') CONFIG.load_file(sys.argv[1]) CONFIG.build_model() test_datasets.load_file(sys.argv[2]) test_datasets.build_model() datesets_model = test_datasets.model initialize_for_running(CONFIG.model.output, CONFIG.model.tf_manager, datesets_model.variables) print("") evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e for e in CONFIG.model.evaluation] for dataset in datesets_model.test_datasets: execution_results, output_data = run_on_dataset( CONFIG.model.tf_manager, CONFIG.model.runners, dataset, CONFIG.model.postprocess, write_out=True) # TODO what if there is no ground truth eval_result = evaluation(evaluators, dataset, CONFIG.model.runners, execution_results, output_data) if eval_result: print_final_evaluation(dataset.name, eval_result)
def run_model(self, dataset: Dataset, write_out: bool = False, log_progress: int = 0) -> Tuple[ List[ExecutionResult], Dict[str, List], Dict[str, List]]: """Run the model on a given dataset. Args: dataset: The dataset on which the model will be executed. write_out: Flag whether the outputs should be printed to a file defined in the dataset object. log_progress: log progress every X seconds Returns: A list of `ExecutionResult`s and a dictionary of the output series. """ if not self._model_built: self.build_model() if not self._vars_loaded: self.load_variables() with self.graph.as_default(): return run_on_dataset( self.model.tf_manager, self.model.runners, self.model.dataset_runner, dataset, self.model.postprocess, write_out=write_out, log_progress=log_progress)
def main(): # pylint: disable=no-member,broad-except if len(sys.argv) != 3: print("Usage: run.py <run_ini_file> <test_datasets>") exit(1) test_datasets = Configuration() test_datasets.add_argument('test_datasets') args, sess = initialize_for_running(sys.argv[1]) datasets_args = test_datasets.load_file(sys.argv[2]) print("") try: for dataset in datasets_args.test_datasets: check_dataset_and_coders(dataset, args.encoders) except Exception as exc: log(str(exc), color='red') exit(1) for dataset in datasets_args.test_datasets: _, _, evaluation = run_on_dataset(sess, args.runner, args.encoders + [args.decoder], args.decoder, dataset, args.evaluation, args.postprocess, write_out=True) if evaluation: print_dataset_evaluation(dataset.name, evaluation)
def post_request(): start_time = datetime.datetime.now() request_data = request.get_json() if request_data is None: response_data = {"error": "No data were provided."} code = 400 else: args = APP.config['args'] sess = APP.config['sess'] try: dataset = Dataset("request", request_data, {}) check_dataset_and_coders(dataset, args.encoders) result, _, _ = run_on_dataset( sess, args.runner, args.encoders + [args.decoder], args.decoder, dataset, args.evaluation, args.postprocess, write_out=True) response_data = {args.decoder.data_id: result} code = 200 #pylint: disable=broad-except except Exception as exc: response_data = {'error': str(exc)} code = 400 response_data['duration'] = (datetime.datetime.now() - start_time).total_seconds() json_response = json.dumps(response_data) response = flask.Response(json_response, content_type='application/json; charset=utf-8') response.headers.add('content-length', len(json_response.encode('utf-8'))) response.status_code = code return response
def run(data): # pragma: no cover args = APP.config["args"] dataset = Dataset("request", data, {}) # TODO check the dataset # check_dataset_and_coders(dataset, args.encoders) _, response_data = run_on_dataset(args.tf_manager, args.runners, dataset, args.postprocess, write_out=False) return response_data
def run_model( self, dataset: Dataset, write_out: bool = False, batch_size: int = None, log_progress: int = 0 ) -> Tuple[List[ExecutionResult], Dict[str, List[Any]]]: """Run the model on a given dataset. Args: dataset: The dataset on which the model will be executed. write_out: Flag whether the outputs should be printed to a file defined in the dataset object. batch_size: size of the minibatch log_progress: log progress every X seconds Returns: A list of `ExecutionResult`s and a dictionary of the output series. """ if not self._model_built: self.build_model() if not self._vars_loaded: self.load_variables() toklevel = self.model.runners_batching_scheme.token_level_batching assert self.model.runners_batching_scheme.batch_bucket_span is None batching_scheme = BatchingScheme(batch_size=batch_size or self.model.runners_batch_size, batch_bucket_span=None, token_level_batching=toklevel, bucketing_ignore_series=[]) with self.graph.as_default(): # TODO: check_dataset_and_coders(dataset, self.model.runners) return run_on_dataset(self.model.tf_manager, self.model.runners, dataset, self.model.postprocess, write_out=write_out, log_progress=log_progress, batching_scheme=batching_scheme)
def main() -> None: # pylint: disable=no-member,broad-except parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", metavar="INI-FILE", help="the configuration file of the experiment") parser.add_argument('datasets', metavar='INI-TEST-DATASETS', help="the configuration of the test datasets") parser.add_argument("-g", "--grid", dest="grid", action="store_true", help="look at the SGE variables for slicing the data") args = parser.parse_args() test_datasets = Configuration() test_datasets.add_argument('test_datasets') test_datasets.add_argument('variables') CONFIG.load_file(args.config) CONFIG.build_model() test_datasets.load_file(args.datasets) test_datasets.build_model() datasets_model = test_datasets.model initialize_for_running(CONFIG.model.output, CONFIG.model.tf_manager, datasets_model.variables) print("") evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e for e in CONFIG.model.evaluation] if args.grid and len(datasets_model.test_datasets) > 1: raise ValueError("Only one test dataset supported when using --grid") for dataset in datasets_model.test_datasets: if args.grid: if ("SGE_TASK_FIRST" not in os.environ or "SGE_TASK_LAST" not in os.environ or "SGE_TASK_STEPSIZE" not in os.environ or "SGE_TASK_ID" not in os.environ): raise EnvironmentError( "Some SGE environment variables are missing") length = int(os.environ["SGE_TASK_STEPSIZE"]) start = int(os.environ["SGE_TASK_ID"]) - 1 end = int(os.environ["SGE_TASK_LAST"]) - 1 if start + length > end: length = end - start + 1 log("Running grid task {} starting at {} with step {}".format( start // length, start, length)) dataset = dataset.subset(start, length) if CONFIG.model.runners_batch_size is None: runners_batch_size = CONFIG.model.batch_size else: runners_batch_size = CONFIG.model.runners_batch_size execution_results, output_data = run_on_dataset( CONFIG.model.tf_manager, CONFIG.model.runners, dataset, CONFIG.model.postprocess, write_out=True, batch_size=runners_batch_size, log_progress=60) # TODO what if there is no ground truth eval_result = evaluation(evaluators, dataset, CONFIG.model.runners, execution_results, output_data) if eval_result: print_final_evaluation(dataset.name, eval_result)