def train_individual_model(run_args):
# Call the training script with the json file as args
# The json file contains all training parameters, including the files for training and validation
# Note: It is necessary to launch a new thread because the command might be prefixed (e. g. use slurm as job
# skeduler to train all folds on different machines
args = run_args["args"]
train_args_json = run_args["json"]
fold_logger = logging.getLogger(f"FOLD {args['id']}")
for handler in fold_logger.handlers:
handler.terminator = ''
for out, err in run(prefix_run_command(
[
sys.executable,
"-u",
args["train_script"],
"--files",
train_args_json,
], args.get("run", None), {"threads": args.get('num_threads', -1)}),
verbose=args.get("verbose", False)):
# Print the output of the thread
if args.get("verbose", False):
if out:
fold_logger.info(out.rstrip("\n"))
if err:
fold_logger.info(err.rstrip("\n"))
return args
def train_individual_model(run_args):
# Call the training script with the json file as args
# The json file contains all training parameters, including the files for training and validation
# Note: It is necessary to launch a new thread because the command might be prefixed (e. g. use slurm as job
# skeduler to train all folds on different machines
args = run_args["args"]
train_args_json = run_args["json"]
for line in run(prefix_run_command([
"python3", "-u",
args["train_script"],
"--files", train_args_json,
], args["run"], {"threads": args['num_threads']}), verbose=args["verbose"]):
# Print the output of the thread
if args["verbose"]:
print(line)
return args
def train_individual_model(run_args):
# Call the training script with the json file as args
# The json file contains all training parameters, including the files for training and validation
# Note: It is necessary to launch a new thread because the command might be prefixed (e. g. use slurm as job
# skeduler to train all folds on different machines
args = run_args["args"]
train_args_json = run_args["json"]
for line in run(prefix_run_command([
"python3", "-u",
args["train_script"],
"--files", train_args_json,
], args["run"], {"threads": args['num_threads']}), verbose=args["verbose"]):
# Print the output of the thread
if args["verbose"]:
print("FOLD {} | {}".format(args["id"], line), end="")
return args
def train_individual_model(run_args):
# Call the training script with the json file as args
# The json file contains all training parameters, including the files for training and validation
# Note: It is necessary to launch a new thread because the command might be prefixed (e. g. use slurm as job
# skeduler to train all folds on different machines
args = run_args["args"]
train_args_json = run_args["json"]
for line in run(prefix_run_command([
sys.executable,
"-u",
args["train_script"],
"--files",
train_args_json,
], args.get("run", None), {"threads": args.get('num_threads', -1)}),
verbose=args.get("verbose", False)):
# Print the output of the thread
if args.get("verbose", False):
print("FOLD {} | {}".format(args["id"], line), end="")
return args
def run_for_single_line(args):
# lines/network/pretraining as base dir
args.base_dir = os.path.join(args.base_dir, "all" if args.n_lines < 0 else str(args.n_lines))
pretrain_prefix = "scratch"
if args.weights and len(args.weights) > 0:
pretrain_prefix = ",".join([split_all_ext(os.path.basename(path))[0] for path in args.weights])
args.base_dir = os.path.join(args.base_dir, args.network, pretrain_prefix)
if not os.path.exists(args.base_dir):
os.makedirs(args.base_dir)
tmp_dir = os.path.join(args.base_dir, "tmp")
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
best_models_dir = os.path.join(args.base_dir, "models")
if not os.path.exists(best_models_dir):
os.makedirs(best_models_dir)
prediction_dir = os.path.join(args.base_dir, "predictions")
if not os.path.exists(prediction_dir):
os.makedirs(prediction_dir)
# select number of files
files = args.train_files
if args.n_lines > 0:
all_files = glob_all(args.train_files)
files = random.sample(all_files, args.n_lines)
# run the cross-fold-training
setattr(args, "max_parallel_models", args.max_parallel_models)
setattr(args, "best_models_dir", best_models_dir)
setattr(args, "temporary_dir", tmp_dir)
setattr(args, "keep_temporary_files", False)
setattr(args, "files", files)
setattr(args, "text_files", None)
setattr(args, "gt_extension", None)
setattr(args, "dataset", DataSetType.FILE)
setattr(args, "best_model_label", "{id}")
if not args.skip_train:
cross_fold_train.main(args)
dump_file = os.path.join(tmp_dir, "prediction.pkl")
# run the prediction
if not args.skip_eval:
# locate the eval script (must be in the same dir as "this")
predict_script_path = os.path.join(this_absdir, "experiment_eval.py")
if len(args.single_fold) > 0:
models = [os.path.join(best_models_dir, "{}.ckpt.json".format(sf)) for sf in args.single_fold]
for m in models:
if not os.path.exists(m):
raise Exception("Expected model at '{}', but file does not exist".format(m))
else:
models = [os.path.join(best_models_dir, d) for d in sorted(os.listdir(best_models_dir)) if d.endswith("json")]
if len(models) != args.n_folds:
raise Exception("Expected {} models, one for each fold respectively, but only {} models were found".format(
args.n_folds, len(models)
))
for line in run(prefix_run_command([
"python3", "-u",
predict_script_path,
"-j", str(args.num_threads),
"--batch_size", str(args.batch_size),
"--dump", dump_file,
"--eval_imgs"] + args.eval_files + [
] + (["--verbose"] if args.verbose else []) + [
"--checkpoint"] + models + [
], args.run, {"threads": args.num_threads}), verbose=args.verbose):
# Print the output of the thread
if args.verbose:
print(line)
import pickle
with open(dump_file, 'rb') as f:
prediction = pickle.load(f)
return prediction
def run_for_single_line(args):
# lines/network/pretraining as base dir
args.base_dir = os.path.join(args.base_dir, "all" if args.n_lines < 0 else str(args.n_lines))
pretrain_prefix = "scratch"
if args.weights and len(args.weights) > 0:
pretrain_prefix = ",".join([split_all_ext(os.path.basename(path))[0] for path in args.weights])
args.base_dir = os.path.join(args.base_dir, args.network, pretrain_prefix)
if not os.path.exists(args.base_dir):
os.makedirs(args.base_dir)
tmp_dir = os.path.join(args.base_dir, "tmp")
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
best_models_dir = os.path.join(args.base_dir, "models")
if not os.path.exists(best_models_dir):
os.makedirs(best_models_dir)
prediction_dir = os.path.join(args.base_dir, "predictions")
if not os.path.exists(prediction_dir):
os.makedirs(prediction_dir)
# select number of files
files = args.train_files
if args.n_lines > 0:
all_files = glob_all(args.train_files)
files = random.sample(all_files, args.n_lines)
# run the cross-fold-training
setattr(args, "max_parallel_models", args.max_parallel_models)
setattr(args, "best_models_dir", best_models_dir)
setattr(args, "temporary_dir", tmp_dir)
setattr(args, "keep_temporary_files", False)
setattr(args, "files", files)
setattr(args, "best_model_label", "{id}")
if not args.skip_train:
cross_fold_train.main(args)
dump_file = os.path.join(tmp_dir, "prediction.pkl")
# run the prediction
if not args.skip_eval:
# locate the eval script (must be in the same dir as "this")
predict_script_path = os.path.join(this_absdir, "experiment_eval.py")
if len(args.single_fold) > 0:
models = [os.path.join(best_models_dir, "{}.ckpt.json".format(sf)) for sf in args.single_fold]
for m in models:
if not os.path.exists(m):
raise Exception("Expected model at '{}', but file does not exist".format(m))
else:
models = [os.path.join(best_models_dir, d) for d in sorted(os.listdir(best_models_dir)) if d.endswith("json")]
if len(models) != args.n_folds:
raise Exception("Expected {} models, one for each fold respectively, but only {} models were found".format(
args.n_folds, len(models)
))
for line in run(prefix_run_command([
"python3", "-u",
predict_script_path,
"-j", str(args.num_threads),
"--batch_size", str(args.batch_size),
"--dump", dump_file,
"--eval_imgs"] + args.eval_files + [
] + (["--verbose"] if args.verbose else []) + [
"--checkpoint"] + models + [
], args.run, {"threads": args.num_threads}), verbose=args.verbose):
# Print the output of the thread
if args.verbose:
print(line)
import pickle
with open(dump_file, 'rb') as f:
prediction = pickle.load(f)
return prediction
def run_for_single_line(args):
# lines/network/pretraining as base dir
args.base_dir = os.path.join(args.base_dir, "all" if args.n_lines < 0 else str(args.n_lines))
pretrain_prefix = "scratch"
if args.weights and len(args.weights) > 0:
pretrain_prefix = ",".join([split_all_ext(os.path.basename(path))[0] for path in args.weights])
args.base_dir = os.path.join(args.base_dir, args.network, pretrain_prefix)
if not os.path.exists(args.base_dir):
os.makedirs(args.base_dir)
tmp_dir = os.path.join(args.base_dir, "tmp")
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
best_models_dir = os.path.join(args.base_dir, "models")
if not os.path.exists(best_models_dir):
os.makedirs(best_models_dir)
prediction_dir = os.path.join(args.base_dir, "predictions")
if not os.path.exists(prediction_dir):
os.makedirs(prediction_dir)
# select number of files
files = args.files
if args.n_lines > 0:
all_files = glob_all(args.files)
files = random.sample(all_files, args.n_lines)
# run the cross-fold-training
setattr(args, "early_stopping_best_model_output_dir", best_models_dir)
setattr(args, "output_dir", tmp_dir)
setattr(args, "keep_temporary_files", False)
setattr(args, "files", files)
setattr(args, "text_files", None)
setattr(args, "gt_extension", None)
setattr(args, "dataset", DataSetType.FILE)
setattr(args, "best_model_label", "{id}")
if not args.skip_train:
train_script.main(args)
dump_file = os.path.join(tmp_dir, "prediction.pkl")
# run the prediction
if not args.skip_eval:
# locate the eval script (must be in the same dir as "this")
predict_script_path = os.path.join(this_absdir, "predict_and_eval.py")
model = os.path.join(best_models_dir, "best.ckpt.json")
if not os.path.exists(model):
raise Exception(f"Expected model at '{model}', but file does not exist")
for line in run(prefix_run_command([
"python3", "-u",
predict_script_path,
"-j", str(args.num_threads),
"--batch_size", str(args.batch_size),
"--dump", dump_file,
"--files"] + args.eval_files + [
] + (["--verbose"] if args.verbose else []) + [
"--checkpoint"] + [model] + [
], args.run, {"threads": args.num_threads}), verbose=args.verbose):
# Print the output of the thread
if args.verbose:
print(line)
import pickle
with open(dump_file, 'rb') as f:
prediction = pickle.load(f)
return prediction
