def _parse_arguments(dataset_path: str, kwargs: dict):
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("max_gap_size") is None:
kwargs["max_gap_size"] = 3
if kwargs.get("min_segment_size") is None:
kwargs["min_segment_size"] = 30
if kwargs.get("smoothing_window") is None:
kwargs["smoothing_window"] = 7
if kwargs.get("poly_order") is None:
kwargs["poly_order"] = 3
if kwargs.get("std_fraction") is None:
kwargs["std_fraction"] = 0.01
if kwargs.get("eigenworms_matrix_path") is None:
kwargs["eigenworms_matrix_path"] = None
if kwargs.get("num_process") is None:
kwargs["num_process"] = os.cpu_count()
if kwargs.get("temp_dir") is None:
kwargs["temp_dir"] = tempfile.gettempdir()
kwargs["temp_dir"] = tempfile.mkdtemp(dir=kwargs["temp_dir"])
dataset_name = get_dataset_name(dataset_path)
kwargs["experiment_dir"] = os.path.join(kwargs["work_dir"], dataset_name)
if kwargs.get("config") is None:
kwargs["config"] = os.path.join(kwargs["experiment_dir"],
CONFIG_FILENAME)
_log_parameters(logger.info, {"dataset_path": dataset_path})
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def _parse_arguments(dataset_path: str, kwargs: dict):
if kwargs.get("temp_dir") is None:
kwargs["temp_dir"] = tempfile.gettempdir()
if kwargs.get("draw_original") is None:
kwargs["draw_original"] = True
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("video_names") is None:
kwargs["video_names"] = None
if kwargs.get("results_file") is None:
kwargs["results_file"] = RESULTS_FILENAME
if kwargs.get("group_name") is None:
kwargs["group_name"] = "resolved"
kwargs["temp_dir"] = tempfile.mkdtemp(dir=kwargs["temp_dir"])
dataset_name = get_dataset_name(dataset_path)
kwargs["experiment_dir"] = os.path.join(kwargs["work_dir"], dataset_name)
if kwargs.get("config") is None:
kwargs["config"] = os.path.join(kwargs["experiment_dir"], CONFIG_FILENAME)
_log_parameters(logger.info, {"dataset_path": dataset_path})
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def _parse_arguments(dataset_path: str, kwargs: dict):
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("batch_size") is None:
kwargs["batch_size"] = 128
if kwargs.get("epochs") is None:
kwargs["epochs"] = 100
if kwargs.get("network_model") is None:
kwargs["network_model"] = model.build_model
if kwargs.get("optimizer") is None:
kwargs["optimizer"] = "adam"
if kwargs.get("loss") is None:
kwargs["loss"] = symmetric_angle_difference
if kwargs.get("random_seed") is None:
kwargs["random_seed"] = None
dataset_name = get_dataset_name(dataset_path)
kwargs["experiment_dir"] = os.path.join(kwargs["work_dir"], dataset_name)
if kwargs.get("config") is None:
kwargs["config"] = os.path.join(kwargs["experiment_dir"],
CONFIG_FILENAME)
_log_parameters(logger.info, {"dataset_path": dataset_path})
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def _parse_arguments(dataset_path: str, kwargs: dict):
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("num_process") is None:
kwargs["num_process"] = os.cpu_count()
if kwargs.get("temp_dir") is None:
kwargs["temp_dir"] = tempfile.gettempdir()
if kwargs.get("batch_size") is None:
kwargs["batch_size"] = 512
if kwargs.get("score_threshold") is None:
kwargs["score_threshold"] = 0.7
if kwargs.get("video_names") is None:
kwargs["video_names"] = None
if kwargs.get("random_seed") is None:
kwargs["random_seed"] = None
kwargs["temp_dir"] = tempfile.mkdtemp(dir=kwargs["temp_dir"])
dataset_name = get_dataset_name(dataset_path)
kwargs["experiment_dir"] = os.path.join(kwargs["work_dir"], dataset_name)
if kwargs.get("model_path") is None:
default_models_dir = os.path.join(kwargs["experiment_dir"],
default_paths.MODELS_DIRS)
kwargs["model_path"] = BestModels(default_models_dir).best_model_path
if kwargs.get("config") is None:
kwargs["config"] = os.path.join(kwargs["experiment_dir"],
CONFIG_FILENAME)
_log_parameters(logger.info, {"dataset_path": dataset_path})
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def _parse_arguments(dataset_path: str, kwargs: dict):
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("video_names") is None:
kwargs["video_names"] = None
dataset_name = get_dataset_name(dataset_path)
kwargs["experiment_dir"] = os.path.join(kwargs["work_dir"], dataset_name)
if kwargs.get("config") is None:
kwargs["config"] = os.path.join(kwargs["experiment_dir"],
CONFIG_FILENAME)
_log_parameters(logger.info, {"dataset_path": dataset_path})
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def _parse_arguments(kwargs: dict):
if kwargs.get("num_samples") is None:
kwargs["num_samples"] = 500
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("theta_dims") is None:
kwargs["theta_dims"] = 100
if kwargs.get("video_names") is None:
kwargs["video_names"] = None
if kwargs.get("save_images") is None:
kwargs["save_images"] = False
if kwargs.get("random_seed") is None:
kwargs["random_seed"] = None
kwargs["resize_options"] = ResizeOptions(**kwargs)
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def calibrate(dataset_loader: str, dataset_path: str, **kwargs):
"""
Calculate the image score for a certain number of labelled frames in the dataset,
this will give an indication on choosing the image similarity threshold when predicting all frames in the dataset.
:param dataset_loader: Name of the dataset loader, for example "tierpsy"
:param dataset_path: Root path of the dataset containing videos of worm
"""
_log_parameters(logger.info, {
"dataset_loader": dataset_loader,
"dataset_path": dataset_path
})
args = _parse_arguments(kwargs)
random.seed(args.random_seed)
np.random.seed(args.random_seed)
dataset_name = get_dataset_name(dataset_path)
experiment_dir = os.path.join(args.work_dir, dataset_name)
calibration_results_dir = os.path.join(
experiment_dir, default_paths.CALIBRATION_RESULTS_DIR)
os.makedirs(calibration_results_dir, exist_ok=True)
dataset = load_dataset(
dataset_loader,
dataset_path,
selected_video_names=args.video_names,
**vars(args),
)
calibrator = _Calibrator(
dataset=dataset,
results_dir=calibration_results_dir,
image_shape=dataset.image_shape,
num_samples=args.num_samples,
theta_dims=args.theta_dims,
)
writer = _ImagesAndScoresWriter(
) if kwargs["save_images"] else _ScoresWriter()
for video_name in dataset.video_names:
results_file = calibrator(video_name=video_name, writer=writer)
yield video_name, results_file
def _parse_arguments(kwargs: dict):
if kwargs.get("num_process") is None:
kwargs["num_process"] = os.cpu_count()
if kwargs.get("temp_dir") is None:
kwargs["temp_dir"] = tempfile.gettempdir()
if kwargs.get("num_train_samples") is None:
kwargs["num_train_samples"] = int(5e5)
if kwargs.get("num_eval_samples") is None:
kwargs["num_eval_samples"] = int(1e4)
if kwargs.get("work_dir") is None:
kwargs["work_dir"] = default_paths.WORK_DIR
if kwargs.get("postures_generation") is None:
kwargs["postures_generation"] = PosturesModel().generate
if kwargs.get("video_names") is None:
kwargs["video_names"] = None
if kwargs.get("random_seed") is None:
kwargs["random_seed"] = None
kwargs["temp_dir"] = tempfile.mkdtemp(dir=kwargs["temp_dir"])
kwargs["resize_options"] = ResizeOptions(**kwargs)
_log_parameters(logger.info, kwargs)
return Namespace(**kwargs)
def generate(dataset_loader: str, dataset_path: str, **kwargs):
"""
Generate synthetic images (training data) and processed real images (evaluation data)
and save them to TFrecord files using multiprocessing
:param dataset_loader: Name of the dataset loader, for example "tierpsy"
:param dataset_path: Root path of the dataset containing videos of worm
"""
_log_parameters(logger.info, {
"dataset_loader": dataset_loader,
"dataset_path": dataset_path
})
args = _parse_arguments(kwargs)
mp.set_start_method("spawn", force=True)
random.seed(args.random_seed)
np.random.seed(args.random_seed)
# setup folders
if not os.path.exists(args.work_dir):
os.mkdir(args.work_dir)
experiment_dir = os.path.join(args.work_dir,
get_dataset_name(dataset_path))
os.makedirs(experiment_dir, exist_ok=True)
tfrecords_dataset_root = os.path.join(experiment_dir,
default_paths.TRAINING_DATA_DIR)
if os.path.exists(tfrecords_dataset_root):
shutil.rmtree(tfrecords_dataset_root)
dataset = load_dataset(
dataset_loader=dataset_loader,
dataset_path=dataset_path,
resize_options=args.resize_options,
selected_video_names=args.video_names,
)
start = time.time()
synthetic_data_generator = SyntheticDataGenerator(
num_process=args.num_process,
temp_dir=args.temp_dir,
dataset=dataset,
postures_generation_fn=args.postures_generation,
enable_random_augmentations=True,
writer=TfrecordLabeledDataWriter,
random_seed=args.random_seed,
)
gen = synthetic_data_generator.generate(
num_samples=args.num_train_samples,
file_pattern=os.path.join(args.temp_dir, SYNTH_TRAIN_DATASET_NAMES),
)
for progress in gen:
yield progress
yield 1.0
theta_dims = len(next(args.postures_generation()))
num_eval_samples = eval_data_generator.generate(
dataset=dataset,
num_samples=args.num_eval_samples,
theta_dims=theta_dims,
file_pattern=os.path.join(args.temp_dir, REAL_EVAL_DATASET_NAMES),
)
shutil.copytree(args.temp_dir, tfrecords_dataset_root)
save_config(
ExperimentConfig(
dataset_loader=dataset_loader,
image_shape=dataset.image_shape,
theta_dimensions=theta_dims,
num_train_samples=args.num_train_samples,
num_eval_samples=num_eval_samples,
resize_factor=args.resize_options.resize_factor,
video_names=dataset.video_names,
),
os.path.join(experiment_dir, CONFIG_FILENAME),
)
end = time.time()
logger.info(f"Done generating training data in : {end - start:.1f}s")