def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): trainer_params, scenario = Trainer.parse_trainer_params(values) parser.add_root_argument("trainer", trainer_params.__class__, default=trainer_params) setattr(namespace, "scenario_cls", scenario)
def main(args=None): parser = TFAIPArgumentParser() parser.add_argument("params_file", type=str, help="path to the trainer_params.json", action=ScenarioSelectionAction) args = parser.parse_args(args=args) with WriteToLogFile(args.trainer.output_dir, append=False): # create the trainer trainer = args.scenario_cls.create_trainer(args.trainer, restore=False) trainer.train()
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): scenario, scenario_params = ScenarioBase.from_path(values) evaluator_params = scenario_params.evaluator parser = TFAIPArgumentParser() parser.add_root_argument("evaluator", evaluator_params.__class__, evaluator_params) setattr(namespace, self.dest, scenario) setattr(namespace, self.dest + "_params", scenario_params)
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): from tfaip.scenario.scenariobase import import_scenario scenario = import_scenario(values) # Now pass the real args of the scenario default_trainer_params = scenario.default_trainer_params() parser.add_root_argument("trainer", default_trainer_params.__class__, default=default_trainer_params) setattr(namespace, self.dest, scenario)
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): from tfaip.imports import ScenarioBase export_dir = values scenario, scenario_params = ScenarioBase.from_path(export_dir) parser.add_root_argument("scenario", scenario_params.__class__, default=scenario_params) setattr(namespace, self.dest, values) setattr(namespace, "scenario_cls", scenario)
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): scenario, scenario_params = ScenarioBase.from_path(values[0]) predict_params = scenario.predictor_cls().params_cls()() parser.add_root_argument("data", scenario.predict_generator_params_cls()) parser.add_root_argument("predict", predict_params.__class__, default=predict_params) setattr(namespace, self.dest, values) setattr(namespace, "scenario", scenario) setattr(namespace, "scenario_params", scenario_params)
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): from tfaip.imports import Trainer output_dir = values trainer_params, scenario = Trainer.parse_trainer_params(output_dir) # parse additional args parser.add_root_argument("trainer", trainer_params.__class__, default=trainer_params) setattr(namespace, self.dest, values) setattr(namespace, "scenario_cls", scenario)
def parse_args(args=None): parser = TFAIPArgumentParser() parser.add_argument( "scenario_selection", help= "Select the scenario by providing the module path which must be in the PYTHONPATH. " "Since a module is expected, separate with dots '.' not slashes. " "The module must either comprise a 'scenario.py' file with one " "scenario, else provide the full path to the Scenario class by separating the class name " "with a ':'. E.g. 'tfaip.scenario.tutorial.min', or " "'tfaip.scenario.tutorial.min.scenario:TutorialScenario'", action=ScenarioSelectionAction, ) args = parser.parse_args(args=args) return args.scenario_selection, args.trainer
def parse_args(args=None): parser = TFAIPArgumentParser() parser.add_argument("--export_dir", required=True, type=str) parser.add_argument("--input_json", required=True, type=str) parser.add_argument("--out", default=None, type=str, help="output folder or .json-file") parser.add_argument("--print", default=False, action="store_true", help="print results to console too") args = parser.parse_args(args=args) return args
def main(args=None): parser = TFAIPArgumentParser() parser.add_argument("output_dir", type=str, help="path to the checkpoint dir to resume from", action=ScenarioSelectionAction) args = parser.parse_args(args=args) with WriteToLogFile(args.trainer.output_dir, append=True): logger.info( "=================================================================" ) logger.info(f"RESUMING TRAINING from {args.trainer.output_dir}") logger.info( "=================================================================" ) # create the trainer trainer = args.scenario_cls.create_trainer(args.trainer, restore=True) trainer.train()
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): from tfaip.imports import ScenarioBase export_dirs = values scenario, scenario_params = ScenarioBase.from_path( export_dirs[0]) # scenario based on first model lav_params = scenario.lav_cls().params_cls()() lav_params.model_path = export_dirs predictor_params = scenario.multi_predictor_cls().params_cls()() parser.add_root_argument("lav", scenario.lav_cls().params_cls(), default=lav_params) parser.add_root_argument("predictor", scenario.multi_predictor_cls().params_cls(), default=predictor_params, ignore=["pipeline"]), parser.add_root_argument("data", scenario.predict_generator_params_cls()) setattr(namespace, self.dest, values) setattr(namespace, "scenario", scenario) setattr(namespace, "scenario_params", scenario_params)
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None): from tfaip.imports import ScenarioBase export_dir = values scenario, scenario_params = ScenarioBase.from_path(export_dir) default_gen_params = scenario.predict_generator_params_cls()() if os.path.exists(os.path.join(export_dir, "trainer_params.json")): # if trainer_params exist load val generator as default with open(os.path.join(export_dir, "trainer_params.json")) as f: p = scenario.trainer_cls().params_cls().from_json(f.read()) default_gen_params = p.gen.lav_gen()[0] lav_params = scenario.lav_cls().params_cls()() lav_params.model_path = export_dir parser.add_root_argument("data", DataGeneratorParams, default=default_gen_params) parser.add_root_argument("lav", scenario.lav_cls().params_cls(), default=lav_params) setattr(namespace, self.dest, values) setattr(namespace, "scenario", scenario) setattr(namespace, "scenario_params", scenario_params)
def parse_args(args=None): parser = TFAIPArgumentParser() parser.add_argument("--export_dir", required=True, nargs="+", action=ScenarioSelectionAction) parser.add_argument("--dump_prediction", type=str, help="Dumps the prediction results as tar.gz") args = parser.parse_args(args=args) return args, args.scenario, args.scenario_params
def parse_args(args=None): parser = TFAIPArgumentParser() parser.add_argument("--prediction", type=str, required=True, help="Path to the prediction dump") parser.add_argument("--scenario", type=str, required=True, action=ScenarioSelectionAction) args = parser.parse_args(args=args) return args.prediction, args.scenario, args.scenario_params
def parse_args(args=None): parser = TFAIPArgumentParser() parser.add_argument("--export_dirs", required=True, nargs="+", action=ScenarioSelectionAction) parser.add_argument( "--run_eagerly", action="store_true", help= "Run the graph in eager mode. This is helpful for debugging. Note that all custom layers must be added to ModelBase!", ) parser.add_argument( "--dump", type=str, help="Dump the predictions and results to the given filepath") args = parser.parse_args(args=args) return args, args.scenario, args.scenario_params, args.predictor
def run(): parser = TFAIPArgumentParser( description="Program to update older graphs to newer version." "Recreate the Graph of a model based on the current code and optional" "changed parameters, then loads the weights, and reexports the model " "with the adapted settings and code." "Note: the weights must be unchanged!" "This is for example useful to adapt a parameter in the stored model, e.g. " "the beam with or a weighting factor: --model.inference_decoder.beam_width=5" ) parser.add_argument("export_dir", action=ScenarioSelectionAction, help="path to the checkpoint dir to resume from") parser.add_argument("--output_dir", help="path where to write the model to") parser.add_argument("--overwrite", help="overwrite the existing model", action="store_true") parser.add_argument( "--no_check_loaded", help="do not check if the loaded weights match with the existing ones." "This is particularly useful if you change the size of the model, " "e.g. by adding a pretrained language model", action="store_true", ) args = parser.parse_args() assert args.output_dir or args.overwrite, "either or required" if args.overwrite: args.output_dir = args.export_dir scenario_cls = args.scenario_cls trainer_params = scenario_cls.default_trainer_params() trainer_params.scenario = args.scenario trainer_params.scenario.print_eval_limit = 0 scenario = trainer_params.scenario.create() # setup training and prediction graph with dummy settings scenario.setup_training("adam", skip_model_load_test=True) def store(path, to_path): assert path is not None assert to_path is not None print(f"Converting from {path} to {to_path}") r = scenario.keras_predict_model.load_weights( os.path.join(path, "serve", "variables", "variables")) if not args.no_check_loaded: r.assert_existing_objects_matched() print( "Variables successfully loaded. All existing objects matched.") else: print( "Skipping verification to check if all variables are present in the provided checkpoint." ) print(f"Attempting to store new model to {to_path}") scenario.export(to_path, export_resources=path != to_path) print(f"{to_path} successfully written.") store(args.export_dir, args.output_dir)
def parse_args(args=None): parser = TFAIPArgumentParser(add_help=True) parser.add_argument("--export_dir", required=True, action=ScenarioSelectionAction) parser.add_argument( "--run_eagerly", action="store_true", help="Run the graph in eager mode. This is helpful for debugging. " "Note that all custom layers must be added to ModelBase!", ) parser.add_argument( "--instantiate_graph", action="store_true", help="Recreate the original graph. This might be required in some edge cases.", ) parser.add_argument("--dump", type=str, help="Dump the predictions and results to the given filepath") args = parser.parse_args(args=args) return args, args.scenario, args.scenario_params