def zscore_video(videofile: Union[str, os.PathLike],
project_config: dict,
stride: int = 10):
"""calculates channel-wise mean and standard deviation for input video.
Calculates mean and std deviation independently for each input video channel. Grayscale videos are converted to RGB.
Saves statistics to the augs/normalization dictionary in project_config. Only takes every STRIDE frames for speed.
Calculates mean and std deviation incrementally to not load thousands of frames into memory at once:
https://notmatthancock.github.io/2017/03/23/simple-batch-stat-updates.html
Args:
videofile: path to video file. Must be one of inputs to file_io/VideoReader: avi, mp4, jpg directory, or hdf5
project_config: dictionary for your deepethogram project. Contains augs/normalization field
stride: only every STRIDE frames will be computed. Use stride=1 for the full video
Returns:
"""
assert os.path.exists(videofile)
assert projects.is_deg_file(videofile)
# config['arch'] = 'flow-generator'
# config['normalization'] = None
# transforms = get_transforms_from_config(config)
# xform = transforms['train']
log.info('zscoring file: {}'.format(videofile))
imdata = get_video_statistics(videofile, stride)
fname = os.path.join(os.path.dirname(videofile), 'stats.yaml')
dictionary = {}
if os.path.isfile(fname):
dictionary = utils.load_yaml(fname)
dictionary['normalization'] = imdata
utils.save_dict_to_yaml(dictionary, fname)
update_project_with_normalization(imdata, project_config)
def main(cfg: DictConfig):
# turn "models" in your project configuration to "full/path/to/models"
cfg = utils.get_absolute_paths_from_cfg(cfg)
log.info('configuration used: ')
log.info(cfg.pretty())
weights = projects.get_weightfile_from_cfg(cfg, model_type='sequence')
assert weights is not None, 'Must either specify a weightfile or use reload.latest=True'
if cfg.sequence.latent_name is None:
# find the latent name used in the weight file you loaded
rundir = os.path.dirname(weights)
loaded_cfg = utils.load_yaml(os.path.join(rundir, 'config.yaml'))
latent_name = loaded_cfg['sequence']['latent_name']
# if this latent name is also None, use the arch of the feature extractor
# this should never happen
if latent_name is None:
latent_name = loaded_cfg['feature_extractor']['arch']
else:
latent_name = cfg.sequence.latent_name
# the output name will be a group in the output hdf5 dataset containing probabilities, etc
if cfg.sequence.output_name is None:
output_name = cfg.sequence.arch
else:
output_name = cfg.sequence.output_name
directory_list = cfg.inference.directory_list
if directory_list is None or len(directory_list) == 0:
raise ValueError('must pass list of directories from commmand line. '
'Ex: directory_list=[path_to_dir1,path_to_dir2] or directory_list=all')
elif type(directory_list) == str and directory_list == 'all':
basedir = cfg.project.data_path
directory_list = utils.get_subfiles(basedir, 'directory')
outputfiles = []
for directory in directory_list:
assert os.path.isdir(directory), 'Not a directory: {}'.format(directory)
record = projects.get_record_from_subdir(directory)
assert record['output'] is not None
outputfiles.append(record['output'])
model = build_model_from_cfg(cfg, 1024, len(cfg.project.class_names))
log.info('model: {}'.format(model))
model = utils.load_weights(model, weights)
metrics_file = os.path.join(os.path.dirname(weights), 'classification_metrics.h5')
with h5py.File(metrics_file, 'r') as f:
thresholds = f['threshold_curves']['val']['optimum'][:]
log.info('thresholds: {}'.format(thresholds))
device = 'cuda:{}'.format(cfg.compute.gpu_id)
class_names = cfg.project.class_names
class_names = np.array(class_names)
extract(model, outputfiles, thresholds, cfg.feature_extractor.final_activation, latent_name, output_name,
cfg.sequence.sequence_length, None, True, device, cfg.inference.ignore_error,
cfg.inference.overwrite, class_names=class_names)
def get_split_from_records(records: dict,
datadir: Union[str, bytes, os.PathLike],
splitfile: Union[str, bytes, os.PathLike] = None,
supervised: bool = True,
reload_split: bool = True,
valid_splits_only: bool = True,
train_val_test: list = [0.7, 0.15, 0.15]):
""" Splits the records into train, validation, and test splits
Parameters
----------
records: dict of dicts
E.g. {'animal': {'rgb': path/to/video.mp4, 'label': path/to/label.csv}, 'animal2': ...}
datadir: str, os.PathLike
absolute path to the base directory containing data. Only used to save split
splitfile: str, os.PathLike
absolute path to file containing a pre-made split to load. If none, make a new one from scratch
supervised: bool
if True, enables the option to use the valid split function
reload_split: bool
if True, tries to load the file in splitfile
valid_splits_only: bool
if True and supervised is True, make sure each split has at least 1 instance of each class
train_val_test: list
fractions / Ns in each split. see train_val_test_split
Returns
-------
split_dictionary: dict
see train_val_test_split
"""
if splitfile is None:
splitfile = os.path.join(datadir, 'split.yaml')
else:
assert os.path.isfile(splitfile), 'split file does not exist! {}'.format(splitfile)
if supervised and valid_splits_only:
# this function makes sure that each split has all classes in the dataset
split_func = get_valid_split
else:
split_func = train_val_test_split
if reload_split and os.path.isfile(splitfile):
split_dictionary = utils.load_yaml(splitfile)
if split_dictionary is None:
# some malformatting
split_dictionary = split_func(records, train_val_test)
# if there are new records, e.g. new records were added to an old splitfile,
# assign them to train, val, or test
split_dictionary = update_split(records, split_dictionary)
else:
split_dictionary = split_func(records, train_val_test)
utils.save_dict_to_yaml(split_dictionary, splitfile)
return split_dictionary
def main(cfg: DictConfig):
assert os.path.isfile(cfg.videofile)
project_config = utils.load_yaml(cfg.project.config_file)
zscore_video(cfg.videofile, project_config, cfg.stride)
def zscore_video(videofile: Union[str, os.PathLike],
project_config: dict,
stride: int = 10):
"""calculates channel-wise mean and standard deviation for input video.
Calculates mean and std deviation independently for each input video channel. Grayscale videos are converted to RGB.
Saves statistics to the augs/normalization dictionary in project_config. Only takes every STRIDE frames for speed.
Calculates mean and std deviation incrementally to not load thousands of frames into memory at once:
https://notmatthancock.github.io/2017/03/23/simple-batch-stat-updates.html
Args:
videofile: path to video file. Must be one of inputs to file_io/VideoReader: avi, mp4, jpg directory, or hdf5
project_config: dictionary for your deepethogram project. Contains augs/normalization field
stride: only every STRIDE frames will be computed. Use stride=1 for the full video
Returns:
"""
assert os.path.isfile(videofile)
assert projects.is_deg_file(videofile)
image_stats = StatsRecorder()
# config['arch'] = 'flow-generator'
# config['normalization'] = None
# transforms = get_transforms_from_config(config)
# xform = transforms['train']
log.info('zscoring file: {}'.format(videofile))
with deepethogram.file_io.VideoReader(videofile) as reader:
log.debug('N frames: {}'.format(len(reader)))
for i in tqdm(range(0, len(reader), stride)):
image = reader[i]
image = image.astype(np.float) / 255
image = image.transpose(2, 1, 0)
# image = image[np.newaxis,...]
# N, C, H, W = image.shape
image = image.reshape(3, -1).transpose(1, 0)
# image = image.reshape(N, C, -1).squeeze().transpose(1, 0)
# if i == 0:
# print(image.shape)
image_stats.update(image)
log.info('final stats: {}'.format(image_stats))
imdata = {
'mean': image_stats.mean,
'std': image_stats.std,
'N': image_stats.nobservations
}
for k, v in imdata.items():
if type(v) == torch.Tensor:
v = v.detach().cpu().numpy()
if type(v) == np.ndarray:
v = v.tolist()
imdata[k] = v
fname = os.path.join(os.path.dirname(videofile), 'stats.yaml')
dictionary = {}
if os.path.isfile(fname):
dictionary = utils.load_yaml(fname)
dictionary['normalization'] = imdata
utils.save_dict_to_yaml(dictionary, fname)
update_project_with_normalization(imdata, project_config)
def sequence_inference(cfg: DictConfig):
cfg = projects.setup_run(cfg)
log.info('args: {}'.format(' '.join(sys.argv)))
# turn "models" in your project configuration to "full/path/to/models"
log.info('configuration used: ')
log.info(OmegaConf.to_yaml(cfg))
weights = projects.get_weightfile_from_cfg(cfg, model_type='sequence')
assert weights is not None, 'Must either specify a weightfile or use reload.latest=True'
run_files = utils.get_run_files_from_weights(weights)
if cfg.sequence.latent_name is None:
# find the latent name used in the weight file you loaded
rundir = os.path.dirname(weights)
loaded_cfg = utils.load_yaml(run_files['config_file'])
latent_name = loaded_cfg['sequence']['latent_name']
# if this latent name is also None, use the arch of the feature extractor
# this should never happen
if latent_name is None:
latent_name = loaded_cfg['feature_extractor']['arch']
else:
latent_name = cfg.sequence.latent_name
if cfg.inference.use_loaded_model_cfg:
output_name = cfg.sequence.output_name
loaded_config_file = run_files['config_file']
loaded_model_cfg = OmegaConf.load(loaded_config_file).sequence
current_model_cfg = cfg.sequence
model_cfg = OmegaConf.merge(current_model_cfg, loaded_model_cfg)
cfg.sequence = model_cfg
# we don't want to use the weights that the trained model was initialized with, but the weights after training
# therefore, overwrite the loaded configuration with the current weights
cfg.sequence.weights = weights
cfg.sequence.latent_name = latent_name
cfg.sequence.output_name = output_name
log.info('latent name used for running sequence inference: {}'.format(latent_name))
# the output name will be a group in the output hdf5 dataset containing probabilities, etc
if cfg.sequence.output_name is None:
output_name = cfg.sequence.arch
else:
output_name = cfg.sequence.output_name
directory_list = cfg.inference.directory_list
if directory_list is None or len(directory_list) == 0:
raise ValueError('must pass list of directories from commmand line. '
'Ex: directory_list=[path_to_dir1,path_to_dir2] or directory_list=all')
elif type(directory_list) == str and directory_list == 'all':
basedir = cfg.project.data_path
directory_list = utils.get_subfiles(basedir, 'directory')
outputfiles = []
for directory in directory_list:
assert os.path.isdir(directory), 'Not a directory: {}'.format(directory)
record = projects.get_record_from_subdir(directory)
assert record['output'] is not None
outputfiles.append(record['output'])
model = build_model_from_cfg(cfg, 1024, len(cfg.project.class_names))
log.info('model: {}'.format(model))
model = utils.load_weights(model, weights)
metrics_file = run_files['metrics_file']
assert os.path.isfile(metrics_file)
best_epoch = utils.get_best_epoch_from_weightfile(weights)
# best_epoch = -1
log.info('best epoch from loaded file: {}'.format(best_epoch))
with h5py.File(metrics_file, 'r') as f:
try:
thresholds = f['val']['metrics_by_threshold']['optimum'][best_epoch, :]
except KeyError:
# backwards compatibility
thresholds = f['threshold_curves']['val']['optimum'][:] # [best_epoch, :]
if thresholds.ndim > 1:
thresholds = thresholds[best_epoch, :]
log.info('thresholds: {}'.format(thresholds))
class_names = list(cfg.project.class_names)
if len(thresholds) != len(class_names):
error_message = '''Number of classes in trained model: {}
Number of classes in project: {}
Did you add or remove behaviors after training this model? If so, please retrain!
'''.format(len(thresholds), len(class_names))
raise ValueError(error_message)
device = 'cuda:{}'.format(cfg.compute.gpu_id)
class_names = cfg.project.class_names
class_names = np.array(class_names)
extract(model,
outputfiles,
thresholds,
cfg.feature_extractor.final_activation,
latent_name,
output_name,
cfg.sequence.sequence_length,
True,
device,
cfg.inference.ignore_error,
cfg.inference.overwrite,
class_names=class_names)