def get_postprocessor_from_cfg(cfg: DictConfig,
thresholds: np.ndarray) -> Type[Postprocessor]:
""" Returns a PostProcessor from an OmegaConf DictConfig returned by a """
if cfg.postprocessor.type is None:
return Postprocessor(thresholds)
elif cfg.postprocessor.type == 'min_bout':
return MinBoutLengthPostprocessor(thresholds,
cfg.postprocessor.min_bout_length)
elif cfg.postprocessor.type == 'min_bout_per_behavior':
if not os.path.isdir(cfg.project.data_path):
cfg = projects.convert_config_paths_to_absolute(cfg)
assert os.path.isdir(cfg.project.data_path)
records = projects.get_records_from_datadir(cfg.project.data_path)
label_list = []
for animal, record in records.items():
labelfile = record['label']
if labelfile is None:
continue
label = file_io.read_labels(labelfile)
# ignore partially labeled videos
if np.any(label == -1):
continue
label_list.append(label)
percentiles = get_bout_length_percentile(
label_list, cfg.postprocessor.min_bout_length)
# percntiles is a dict: keys are behaviors, values are percentiles
# need to round and then cast to int
percentiles = np.round(np.array(list(
percentiles.values()))).astype(int)
return MinBoutLengthPerBehaviorPostprocessor(thresholds, percentiles)
else:
raise NotImplementedError
def read_all_labels(labelfiles: list):
""" Function for reading all labels into memory """
labels = []
for i, labelfile in enumerate(labelfiles):
assert (os.path.isfile(labelfile))
label_type = os.path.splitext(labelfile)[1][1:]
# labelfile, label_type = find_labelfile(video)
label = read_labels(labelfile)
H, W = label.shape
# labels should be time x num_behaviors
if W > H:
label = label.T
if label.shape[1] == 1:
# add a background class
warnings.warn('binary labels found, adding background class')
label = np.hstack((np.logical_not(label), label))
labels.append(label)
label_no_ignores = np.copy(label)
label_no_ignores[label_no_ignores == -1] = 0
if i == 0:
class_counts = label_no_ignores.sum(axis=0)
num_pos = (label == 1).sum(axis=0)
num_neg = (label == 0).sum(axis=0)
else:
class_counts += label_no_ignores.sum(axis=0)
num_pos += (label == 1).sum(axis=0)
num_neg += (label == 0).sum(axis=0)
num_labels = len(labels)
labels = np.concatenate(labels)
class_counts = class_counts
return labels, class_counts, num_labels, num_pos, num_neg
def print_dataset_info(datadir: Union[str, os.PathLike]) -> None:
"""Prints information about your dataset.
- video path
- number of unlabeled rows in a video
- number of examples of each behavior in each video
Parameters
----------
datadir : Union[str, os.PathLike]
[description]
"""
records = projects.get_records_from_datadir(datadir)
for key, record in records.items():
log.info('Information about subdir {}'.format(key))
if record['rgb'] is not None:
log.info('Video: {}'.format(record['rgb']))
if record['label'] is not None:
label = file_io.read_labels(record['label'])
if np.sum(label == -1) > 0:
unlabeled_rows = np.any(label == -1, axis=0)
n_unlabeled = np.sum(unlabeled_rows)
log.warning('{} UNLABELED ROWS!'.format(n_unlabeled) + \
'VIDEO WILL NOT BE USED FOR FEATURE_EXTRACTOR OR SEQUENCE TRAINING.')
else:
class_counts = label.sum(axis=0)
log.info('Labels with counts: {}'.format(class_counts))
def get_val_n_labels(split_dict: dict, labelfiles: dict):
valfiles = split_dict['val']
sums = []
for key in valfiles:
labelfile = labelfiles['val'][key]
label = read_labels(labelfile)
sums.append(label.sum(axis=0))
sums = np.stack(sums)
return np.sum(sums, axis=0)
def purge_unlabeled_videos(video_list: list, label_list: list) -> Tuple[list, list]:
"""Get rid of any videos that contain unlabeled frames.
Goes through all label files, loads them. If they contain any -1 values, remove both the video and the label
from their respective lists
"""
valid_videos = []
valid_labels = []
warning_string = '''Labelfile {} associated with video {} has unlabeled frames!
Please finish labeling or click the Finalize Labels button on the GUI.'''
for i in range(len(label_list)):
label = read_labels(label_list[i])
has_unlabeled_frames = np.any(label == -1)
if not has_unlabeled_frames:
valid_videos.append(video_list[i])
valid_labels.append(label_list[i])
else:
log.warning(warning_string.format(label_list[i], video_list[i]))
return video_list, label_list
def purge_unlabeled_elements_from_records(records: dict) -> dict:
valid_records = {}
warning_message = '''labelfile {} has unlabeled frames!
Please finish labeling or click the Finalize Labels button on the GUI.
Associated files: {}'''
for animal, record in records.items():
labelfile = record['label']
if labelfile is None:
log.warning('Record {} does not have a labelfile! Please start and finish labeling. '.format(animal) + \
'Associated files: {}'.format(record))
continue
label = read_labels(labelfile)
has_unlabeled_frames = np.any(label == -1)
if has_unlabeled_frames:
log.warning(warning_message.format(animal, record))
else:
valid_records[animal] = record
return valid_records
def extract(rgbs: list,
model,
final_activation: str,
thresholds: np.ndarray,
postprocessor,
mean_by_channels,
fusion: str,
num_rgb: int,
latent_name: str,
class_names: list = ['background'],
device: str = 'cuda:0',
cpu_transform=None,
gpu_transform=None,
ignore_error=True,
overwrite=False,
num_workers: int = 1,
batch_size: int = 1):
""" Use the model to extract spatial and flow feature vectors, and predictions, and save them to disk.
Assumes you have a pretrained model, and K classes. Will go through each video in rgbs, run inference, extracting
the 512-d spatial features, 512-d flow features, K-d probabilities to disk for each video frame.
Also stores thresholds for later reloading.
Output file structure (outputs.h5):
- latent_name
- spatial_features: (T x 512) neural activations from before the last fully connected layer of the spatial
model
- flow_features: (T x 512) neural activations from before the last fully connected layer of the flow model
- logits: (T x K) unnormalized logits from after the fusion layer
- P: (T x K) values after the activation function (specified by final_activation)
- thresholds: (K,) loaded thresholds that convert probabilities to binary predictions
- class_names: (K,) loaded class names for your project
Args:
rgbs (list): list of input video files
model (nn.Module): a hidden-two-stream deepethogram model
see deepethogram/feature_extractor/models/hidden_two_stream.py
final_activation (str): one of sigmoid or softmax
thresholds (np.ndarray): array of shape (K,), thresholds between 0 and 1 that turns probabilities into
binary predictions
fusion (str): one of [average, concatenate]
num_rgb (int): number of input images to your model
latent_name (str): an HDF5 group with this name will be in your output HDF5 file.
class_names (list): a list of class names. Will be saved so that this HDF5 file can be read without any project
configuration files
device (str): cuda device on which models will be run
transform (transforms.Compose): data augmentation. Since this is inference, should only include resizing,
cropping, and normalization
ignore_error (bool): if True, an error on one video will not stop inference
overwrite (bool): if an HDF5 group with the given latent_name is present in the HDF5 file:
if True, overwrites data with current values. if False, skips that video
"""
# make sure we're using CUDNN for speed
torch.backends.cudnn.benchmark = True
assert isinstance(model, torch.nn.Module)
device = torch.device(device)
if device.type != 'cpu':
torch.cuda.set_device(device)
model = model.to(device)
# freeze model and set to eval mode for batch normalization
model.set_mode('inference')
# double checknig
for parameter in model.parameters():
parameter.requires_grad = False
model.eval()
if final_activation == 'softmax':
activation_function = nn.Softmax(dim=1)
elif final_activation == 'sigmoid':
activation_function = nn.Sigmoid()
else:
raise ValueError(
'unknown final activation: {}'.format(final_activation))
# 16 is a decent trade off between CPU and GPU load on datasets I've tested
if batch_size == 'auto':
batch_size = 16
batch_size = min(batch_size, 16)
log.info('inference batch size: {}'.format(batch_size))
class_names = [n.encode("ascii", "ignore") for n in class_names]
log.debug('model training mode: {}'.format(model.training))
# iterate over movie files
for i in tqdm(range(len(rgbs))):
rgb = rgbs[i]
basename = os.path.splitext(rgb)[0]
# make the outputfile have the same name as the video, with _outputs appended
h5file = basename + '_outputs.h5'
mode = 'r+' if os.path.isfile(h5file) else 'w'
should_run = check_if_should_run_inference(h5file, mode, latent_name,
overwrite)
if not should_run:
continue
# iterate over each frame of the movie
outputs = predict_single_video(rgb,
model,
activation_function,
fusion,
num_rgb,
mean_by_channels,
device,
cpu_transform,
gpu_transform,
should_print=i == 0,
num_workers=num_workers,
batch_size=batch_size)
if i == 0:
for k, v in outputs.items():
log.info('{}: {}'.format(k, v.shape))
if k == 'debug':
log.debug(
'All should be 1.0: min: {:.4f} mean {:.4f} max {:.4f}'
.format(v.min(), v.mean(), v.max()))
# if running inference from multiple processes, this will wait until the resource is available
has_worked = False
while not has_worked:
try:
f = h5py.File(h5file, 'r+')
except OSError as e:
log.warning('resource unavailable, waiting 30 seconds...')
time.sleep(30)
else:
has_worked = True
try:
predictions = postprocessor(
outputs['probabilities'].detach().cpu().numpy())
labelfile = projects.find_labelfiles(os.path.dirname(rgb))[0]
labels = read_labels(labelfile)
f1 = f1_score(labels, predictions, average='macro')
log.info('macro F1: {}'.format(f1))
except Exception as e:
log.warning('error calculating f1: {}'.format(e))
# since this is just for debugging, ignore
pass
# these assignments are where it's actually saved to disk
group = f.create_group(latent_name)
group.create_dataset('thresholds', data=thresholds, dtype=np.float32)
group.create_dataset('logits',
data=outputs['logits'],
dtype=np.float32)
group.create_dataset('P',
data=outputs['probabilities'],
dtype=np.float32)
group.create_dataset('spatial_features',
data=outputs['spatial_features'],
dtype=np.float32)
group.create_dataset('flow_features',
data=outputs['flow_features'],
dtype=np.float32)
dt = h5py.string_dtype()
group.create_dataset('class_names', data=class_names, dtype=dt)
del outputs
f.close()