def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
trainer_params, scenario = Trainer.parse_trainer_params(values)
parser.add_root_argument("trainer",
trainer_params.__class__,
default=trainer_params)
setattr(namespace, "scenario_cls", scenario)
def main(args=None):
parser = TFAIPArgumentParser()
parser.add_argument("params_file",
type=str,
help="path to the trainer_params.json",
action=ScenarioSelectionAction)
args = parser.parse_args(args=args)
with WriteToLogFile(args.trainer.output_dir, append=False):
# create the trainer
trainer = args.scenario_cls.create_trainer(args.trainer, restore=False)
trainer.train()
def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
scenario, scenario_params = ScenarioBase.from_path(values)
evaluator_params = scenario_params.evaluator
parser = TFAIPArgumentParser()
parser.add_root_argument("evaluator", evaluator_params.__class__,
evaluator_params)
setattr(namespace, self.dest, scenario)
setattr(namespace, self.dest + "_params", scenario_params)
def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
from tfaip.scenario.scenariobase import import_scenario
scenario = import_scenario(values)
# Now pass the real args of the scenario
default_trainer_params = scenario.default_trainer_params()
parser.add_root_argument("trainer",
default_trainer_params.__class__,
default=default_trainer_params)
setattr(namespace, self.dest, scenario)
def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
from tfaip.imports import ScenarioBase
export_dir = values
scenario, scenario_params = ScenarioBase.from_path(export_dir)
parser.add_root_argument("scenario",
scenario_params.__class__,
default=scenario_params)
setattr(namespace, self.dest, values)
setattr(namespace, "scenario_cls", scenario)
def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
scenario, scenario_params = ScenarioBase.from_path(values[0])
predict_params = scenario.predictor_cls().params_cls()()
parser.add_root_argument("data",
scenario.predict_generator_params_cls())
parser.add_root_argument("predict",
predict_params.__class__,
default=predict_params)
setattr(namespace, self.dest, values)
setattr(namespace, "scenario", scenario)
setattr(namespace, "scenario_params", scenario_params)
def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
from tfaip.imports import Trainer
output_dir = values
trainer_params, scenario = Trainer.parse_trainer_params(output_dir)
# parse additional args
parser.add_root_argument("trainer",
trainer_params.__class__,
default=trainer_params)
setattr(namespace, self.dest, values)
setattr(namespace, "scenario_cls", scenario)
def parse_args(args=None):
parser = TFAIPArgumentParser()
parser.add_argument(
"scenario_selection",
help=
"Select the scenario by providing the module path which must be in the PYTHONPATH. "
"Since a module is expected, separate with dots '.' not slashes. "
"The module must either comprise a 'scenario.py' file with one "
"scenario, else provide the full path to the Scenario class by separating the class name "
"with a ':'. E.g. 'tfaip.scenario.tutorial.min', or "
"'tfaip.scenario.tutorial.min.scenario:TutorialScenario'",
action=ScenarioSelectionAction,
)
args = parser.parse_args(args=args)
return args.scenario_selection, args.trainer
def parse_args(args=None):
parser = TFAIPArgumentParser()
parser.add_argument("--export_dir", required=True, type=str)
parser.add_argument("--input_json", required=True, type=str)
parser.add_argument("--out",
default=None,
type=str,
help="output folder or .json-file")
parser.add_argument("--print",
default=False,
action="store_true",
help="print results to console too")
args = parser.parse_args(args=args)
return args
def main(args=None):
parser = TFAIPArgumentParser()
parser.add_argument("output_dir",
type=str,
help="path to the checkpoint dir to resume from",
action=ScenarioSelectionAction)
args = parser.parse_args(args=args)
with WriteToLogFile(args.trainer.output_dir, append=True):
logger.info(
"================================================================="
)
logger.info(f"RESUMING TRAINING from {args.trainer.output_dir}")
logger.info(
"================================================================="
)
# create the trainer
trainer = args.scenario_cls.create_trainer(args.trainer, restore=True)
trainer.train()
def __call__(self,
parser: TFAIPArgumentParser,
namespace,
values,
option_string=None):
from tfaip.imports import ScenarioBase
export_dirs = values
scenario, scenario_params = ScenarioBase.from_path(
export_dirs[0]) # scenario based on first model
lav_params = scenario.lav_cls().params_cls()()
lav_params.model_path = export_dirs
predictor_params = scenario.multi_predictor_cls().params_cls()()
parser.add_root_argument("lav",
scenario.lav_cls().params_cls(),
default=lav_params)
parser.add_root_argument("predictor",
scenario.multi_predictor_cls().params_cls(),
default=predictor_params,
ignore=["pipeline"]),
parser.add_root_argument("data",
scenario.predict_generator_params_cls())
setattr(namespace, self.dest, values)
setattr(namespace, "scenario", scenario)
setattr(namespace, "scenario_params", scenario_params)
def __call__(self, parser: TFAIPArgumentParser, namespace, values, option_string=None):
from tfaip.imports import ScenarioBase
export_dir = values
scenario, scenario_params = ScenarioBase.from_path(export_dir)
default_gen_params = scenario.predict_generator_params_cls()()
if os.path.exists(os.path.join(export_dir, "trainer_params.json")):
# if trainer_params exist load val generator as default
with open(os.path.join(export_dir, "trainer_params.json")) as f:
p = scenario.trainer_cls().params_cls().from_json(f.read())
default_gen_params = p.gen.lav_gen()[0]
lav_params = scenario.lav_cls().params_cls()()
lav_params.model_path = export_dir
parser.add_root_argument("data", DataGeneratorParams, default=default_gen_params)
parser.add_root_argument("lav", scenario.lav_cls().params_cls(), default=lav_params)
setattr(namespace, self.dest, values)
setattr(namespace, "scenario", scenario)
setattr(namespace, "scenario_params", scenario_params)
def parse_args(args=None):
parser = TFAIPArgumentParser()
parser.add_argument("--export_dir",
required=True,
nargs="+",
action=ScenarioSelectionAction)
parser.add_argument("--dump_prediction",
type=str,
help="Dumps the prediction results as tar.gz")
args = parser.parse_args(args=args)
return args, args.scenario, args.scenario_params
def parse_args(args=None):
parser = TFAIPArgumentParser()
parser.add_argument("--prediction",
type=str,
required=True,
help="Path to the prediction dump")
parser.add_argument("--scenario",
type=str,
required=True,
action=ScenarioSelectionAction)
args = parser.parse_args(args=args)
return args.prediction, args.scenario, args.scenario_params
def parse_args(args=None):
parser = TFAIPArgumentParser()
parser.add_argument("--export_dirs",
required=True,
nargs="+",
action=ScenarioSelectionAction)
parser.add_argument(
"--run_eagerly",
action="store_true",
help=
"Run the graph in eager mode. This is helpful for debugging. Note that all custom layers must be added to ModelBase!",
)
parser.add_argument(
"--dump",
type=str,
help="Dump the predictions and results to the given filepath")
args = parser.parse_args(args=args)
return args, args.scenario, args.scenario_params, args.predictor
def run():
parser = TFAIPArgumentParser(
description="Program to update older graphs to newer version."
"Recreate the Graph of a model based on the current code and optional"
"changed parameters, then loads the weights, and reexports the model "
"with the adapted settings and code."
"Note: the weights must be unchanged!"
"This is for example useful to adapt a parameter in the stored model, e.g. "
"the beam with or a weighting factor: --model.inference_decoder.beam_width=5"
)
parser.add_argument("export_dir",
action=ScenarioSelectionAction,
help="path to the checkpoint dir to resume from")
parser.add_argument("--output_dir",
help="path where to write the model to")
parser.add_argument("--overwrite",
help="overwrite the existing model",
action="store_true")
parser.add_argument(
"--no_check_loaded",
help="do not check if the loaded weights match with the existing ones."
"This is particularly useful if you change the size of the model, "
"e.g. by adding a pretrained language model",
action="store_true",
)
args = parser.parse_args()
assert args.output_dir or args.overwrite, "either or required"
if args.overwrite:
args.output_dir = args.export_dir
scenario_cls = args.scenario_cls
trainer_params = scenario_cls.default_trainer_params()
trainer_params.scenario = args.scenario
trainer_params.scenario.print_eval_limit = 0
scenario = trainer_params.scenario.create()
# setup training and prediction graph with dummy settings
scenario.setup_training("adam", skip_model_load_test=True)
def store(path, to_path):
assert path is not None
assert to_path is not None
print(f"Converting from {path} to {to_path}")
r = scenario.keras_predict_model.load_weights(
os.path.join(path, "serve", "variables", "variables"))
if not args.no_check_loaded:
r.assert_existing_objects_matched()
print(
"Variables successfully loaded. All existing objects matched.")
else:
print(
"Skipping verification to check if all variables are present in the provided checkpoint."
)
print(f"Attempting to store new model to {to_path}")
scenario.export(to_path, export_resources=path != to_path)
print(f"{to_path} successfully written.")
store(args.export_dir, args.output_dir)
def parse_args(args=None):
parser = TFAIPArgumentParser(add_help=True)
parser.add_argument("--export_dir", required=True, action=ScenarioSelectionAction)
parser.add_argument(
"--run_eagerly",
action="store_true",
help="Run the graph in eager mode. This is helpful for debugging. "
"Note that all custom layers must be added to ModelBase!",
)
parser.add_argument(
"--instantiate_graph",
action="store_true",
help="Recreate the original graph. This might be required in some edge cases.",
)
parser.add_argument("--dump", type=str, help="Dump the predictions and results to the given filepath")
args = parser.parse_args(args=args)
return args, args.scenario, args.scenario_params