def write(self, filename: str, source_object: dict):
json_dumps(source_object, filename)
def write(
cls,
filename: str,
source_object: object,
append: bool = False,
save_frame_data: bool = False,
frame_data_format: str = "png",
all_labeled: bool = False,
suggested: bool = False,
):
labels = source_object
# Delete the file if it exists, we want to start from scratch since
# h5py truncates the file which seems to not actually delete data
# from the file. Don't if we are appending of course.
if os.path.exists(filename) and not append:
os.unlink(filename)
# Serialize all the meta-data to JSON.
d = labels.to_dict(skip_labels=True)
if save_frame_data:
new_videos = labels.save_frame_data_hdf5(
filename,
format=frame_data_format,
user_labeled=True,
all_labeled=all_labeled,
suggested=suggested,
)
# Replace path to video file with "." (which indicates that the
# video is in the same file as the HDF5 labels dataset).
# Otherwise, the video paths will break if the HDF5 labels
# dataset file is moved.
for vid in new_videos:
vid.backend.filename = "."
d["videos"] = Video.cattr().unstructure(new_videos)
with h5py.File(filename, "a") as f:
# Add all the JSON metadata
meta_group = f.require_group("metadata")
meta_group.attrs["format_id"] = cls.FORMAT_ID
# If we are appending and there already exists JSON metadata
if append and "json" in meta_group.attrs:
# Otherwise, we need to read the JSON and append to the lists
old_labels = labels_json.LabelsJsonAdaptor.from_json_data(
meta_group.attrs["json"].tostring().decode()
)
# A function to join to list but only include new non-dupe entries
# from the right hand list.
def append_unique(old, new):
unique = []
for x in new:
try:
matches = [y.matches(x) for y in old]
except AttributeError:
matches = [x == y for y in old]
# If there were no matches, this is a unique object.
if sum(matches) == 0:
unique.append(x)
else:
# If we have an object that matches, replace the instance
# with the one from the new list. This will will make sure
# objects on the Instances are the same as those in the
# Labels lists.
for i, match in enumerate(matches):
if match:
old[i] = x
return old + unique
# Append the lists
labels.tracks = append_unique(old_labels.tracks, labels.tracks)
labels.skeletons = append_unique(old_labels.skeletons, labels.skeletons)
labels.videos = append_unique(old_labels.videos, labels.videos)
labels.nodes = append_unique(old_labels.nodes, labels.nodes)
# FIXME: Do something for suggestions and negative_anchors
# Get the dict for JSON and save it over the old data
d = labels.to_dict(skip_labels=True)
if not append:
# These items are stored in separate lists because the metadata
# group got to be too big.
for key in ("videos", "tracks", "suggestions"):
# Convert for saving in hdf5 dataset
data = [np.string_(json_dumps(item)) for item in d[key]]
hdf5_key = f"{key}_json"
# Save in its own dataset (e.g., videos_json)
f.create_dataset(hdf5_key, data=data, maxshape=(None,))
# Clear from dict since we don't want to save this in attribute
d[key] = []
# Output the dict to JSON
meta_group.attrs["json"] = np.string_(json_dumps(d))
# FIXME: We can probably construct these from attrs fields
# We will store Instances and PredcitedInstances in the same
# table. instance_type=0 or Instance and instance_type=1 for
# PredictedInstance, score will be ignored for Instances.
instance_dtype = np.dtype(
[
("instance_id", "i8"),
("instance_type", "u1"),
("frame_id", "u8"),
("skeleton", "u4"),
("track", "i4"),
("from_predicted", "i8"),
("score", "f4"),
("point_id_start", "u8"),
("point_id_end", "u8"),
]
)
frame_dtype = np.dtype(
[
("frame_id", "u8"),
("video", "u4"),
("frame_idx", "u8"),
("instance_id_start", "u8"),
("instance_id_end", "u8"),
]
)
num_instances = len(labels.all_instances)
max_skeleton_size = max([len(s.nodes) for s in labels.skeletons], default=0)
# Initialize data arrays for serialization
points = np.zeros(num_instances * max_skeleton_size, dtype=Point.dtype)
pred_points = np.zeros(
num_instances * max_skeleton_size, dtype=PredictedPoint.dtype
)
instances = np.zeros(num_instances, dtype=instance_dtype)
frames = np.zeros(len(labels), dtype=frame_dtype)
# Pre compute some structures to make serialization faster
skeleton_to_idx = {
skeleton: labels.skeletons.index(skeleton)
for skeleton in labels.skeletons
}
track_to_idx = {
track: labels.tracks.index(track) for track in labels.tracks
}
track_to_idx[None] = -1
video_to_idx = {
video: labels.videos.index(video) for video in labels.videos
}
instance_type_to_idx = {Instance: 0, PredictedInstance: 1}
# Each instance we create will have and index in the dataset, keep track of
# these so we can quickly add from_predicted links on a second pass.
instance_to_idx = {}
instances_with_from_predicted = []
instances_from_predicted = []
# If we are appending, we need look inside to see what frame, instance, and
# point ids we need to start from. This gives us offsets to use.
if append and "points" in f:
point_id_offset = f["points"].shape[0]
pred_point_id_offset = f["pred_points"].shape[0]
instance_id_offset = f["instances"][-1]["instance_id"] + 1
frame_id_offset = int(f["frames"][-1]["frame_id"]) + 1
else:
point_id_offset = 0
pred_point_id_offset = 0
instance_id_offset = 0
frame_id_offset = 0
point_id = 0
pred_point_id = 0
instance_id = 0
for frame_id, label in enumerate(labels):
frames[frame_id] = (
frame_id + frame_id_offset,
video_to_idx[label.video],
label.frame_idx,
instance_id + instance_id_offset,
instance_id + instance_id_offset + len(label.instances),
)
for instance in label.instances:
# Add this instance to our lookup structure we will need for
# from_predicted links
instance_to_idx[instance] = instance_id
parray = instance.get_points_array(copy=False, full=True)
instance_type = type(instance)
# Check whether we are working with a PredictedInstance or an
# Instance.
if instance_type is PredictedInstance:
score = instance.score
pid = pred_point_id + pred_point_id_offset
else:
score = np.nan
pid = point_id + point_id_offset
# Keep track of any from_predicted instance links, we will
# insert the correct instance_id in the dataset after we are
# done.
if instance.from_predicted:
instances_with_from_predicted.append(instance_id)
instances_from_predicted.append(instance.from_predicted)
# Copy all the data
instances[instance_id] = (
instance_id + instance_id_offset,
instance_type_to_idx[instance_type],
frame_id,
skeleton_to_idx[instance.skeleton],
track_to_idx[instance.track],
-1,
score,
pid,
pid + len(parray),
)
# If these are predicted points, copy them to the predicted point
# array otherwise, use the normal point array
if type(parray) is PredictedPointArray:
pred_points[
pred_point_id : (pred_point_id + len(parray))
] = parray
pred_point_id = pred_point_id + len(parray)
else:
points[point_id : (point_id + len(parray))] = parray
point_id = point_id + len(parray)
instance_id = instance_id + 1
# Add from_predicted links
for instance_id, from_predicted in zip(
instances_with_from_predicted, instances_from_predicted
):
try:
instances[instance_id]["from_predicted"] = instance_to_idx[
from_predicted
]
except KeyError:
# If we haven't encountered the from_predicted instance yet then
# don't save the link. It's possible for a user to create a regular
# instance from a predicted instance and then delete all predicted
# instances from the file, but in this case I don’t think there's
# any reason to remember which predicted instance the regular
# instance came from.
pass
# We pre-allocated our points array with max possible size considering the
# max skeleton size, drop any unused points.
points = points[0:point_id]
pred_points = pred_points[0:pred_point_id]
# Create datasets if we need to
if append and "points" in f:
f["points"].resize((f["points"].shape[0] + points.shape[0]), axis=0)
f["points"][-points.shape[0] :] = points
f["pred_points"].resize(
(f["pred_points"].shape[0] + pred_points.shape[0]), axis=0
)
f["pred_points"][-pred_points.shape[0] :] = pred_points
f["instances"].resize(
(f["instances"].shape[0] + instances.shape[0]), axis=0
)
f["instances"][-instances.shape[0] :] = instances
f["frames"].resize((f["frames"].shape[0] + frames.shape[0]), axis=0)
f["frames"][-frames.shape[0] :] = frames
else:
f.create_dataset(
"points", data=points, maxshape=(None,), dtype=Point.dtype
)
f.create_dataset(
"pred_points",
data=pred_points,
maxshape=(None,),
dtype=PredictedPoint.dtype,
)
f.create_dataset(
"instances", data=instances, maxshape=(None,), dtype=instance_dtype
)
f.create_dataset(
"frames", data=frames, maxshape=(None,), dtype=frame_dtype
)
def write(
cls,
filename: str,
source_object: str,
compress: Optional[bool] = None,
save_frame_data: bool = False,
frame_data_format: str = "png",
):
"""
Save a Labels instance to a JSON format.
Args:
filename: The filename to save the data to.
source_object: The labels dataset to save.
compress: Whether the data be zip compressed or not? If True,
the JSON will be compressed using Python's shutil.make_archive
command into a PKZIP zip file. If compress is True then
filename will have a .zip appended to it.
save_frame_data: Whether to save the image data for each frame.
For each video in the dataset, all frames that have labels
will be stored as an imgstore dataset.
If save_frame_data is True then compress will be forced to True
since the archive must contain both the JSON data and image
data stored in ImgStores.
frame_data_format: If save_frame_data is True, then this argument
is used to set the data format to use when writing frame
data to ImgStore objects. Supported formats should be:
* 'pgm',
* 'bmp',
* 'ppm',
* 'tif',
* 'png',
* 'jpg',
* 'npy',
* 'mjpeg/avi',
* 'h264/mkv',
* 'avc1/mp4'
Note: 'h264/mkv' and 'avc1/mp4' require separate installation
of these codecs on your system. They are excluded from SLEAP
because of their GPL license.
Returns:
None
"""
labels = source_object
if compress is None:
compress = filename.endswith(".zip")
# Lets make a temporary directory to store the image frame data or pre-compressed json
# in case we need it.
with tempfile.TemporaryDirectory() as tmp_dir:
# If we are saving frame data along with the datasets. We will replace videos with
# new video object that represent video data from just the labeled frames.
if save_frame_data:
# Create a set of new Video objects with imgstore backends. One for each
# of the videos. We will only include the labeled frames though. We will
# then replace each video with this new video
new_videos = labels.save_frame_data_imgstore(
output_dir=tmp_dir, format=frame_data_format)
# Make video paths relative
for vid in new_videos:
tmp_path = vid.filename
# Get the parent dir of the YAML file.
# Use "/" since this works on Windows and posix
img_store_dir = (
os.path.basename(os.path.split(tmp_path)[0]) + "/" +
os.path.basename(tmp_path))
# Change to relative path
vid.backend.filename = img_store_dir
# Convert to a dict, not JSON yet, because we need to patch up the videos
d = labels.to_dict()
d["videos"] = Video.cattr().unstructure(new_videos)
else:
d = labels.to_dict()
# Set file format version
d["format_id"] = cls.FORMAT_ID
if compress or save_frame_data:
# Ensure that filename ends with .json
# shutil will append .zip
filename = re.sub("(\\.json)?(\\.zip)?$", ".json", filename)
# Write the json to the tmp directory, we will zip it up with the frame data.
full_out_filename = os.path.join(tmp_dir,
os.path.basename(filename))
json_dumps(d, full_out_filename)
# Create the archive
shutil.make_archive(base_name=filename,
root_dir=tmp_dir,
format="zip")
# If the user doesn't want to compress, then just write the json to the filename
else:
json_dumps(d, filename)
def to_hdf5(
self,
path: str,
dataset: str,
frame_numbers: List[int] = None,
format: str = "",
index_by_original: bool = True,
):
"""Convert frames from arbitrary video backend to HDF5Video.
Used for building an HDF5 that holds all data needed for training.
Args:
path: Filename to HDF5 (which could already exist).
dataset: The HDF5 dataset in which to store video frames.
frame_numbers: A list of frame numbers from the video to save.
If None save the entire video.
format: If non-empty, then encode images in format before saving.
Otherwise, save numpy matrix of frames.
index_by_original: If the index_by_original is set to True then
the get_frame function will accept the original frame
numbers of from original video.
If False, then it will accept the frame index directly.
Default to True so that we can use resulting video in a
dataset to replace another video without having to update
all the frame indices in the dataset.
Returns:
A new Video object that references the HDF5 dataset.
"""
# If the user has not provided a list of frames to store, store them all.
if frame_numbers is None:
frame_numbers = range(self.num_frames)
if frame_numbers:
frame_data = self.get_frames(frame_numbers)
else:
frame_data = np.zeros((1, 1, 1, 1))
frame_numbers_data = np.array(list(frame_numbers), dtype=int)
with h5.File(path, "a") as f:
if format:
def encode(img):
_, encoded = cv2.imencode("." + format, img)
return np.squeeze(encoded)
dtype = h5.special_dtype(vlen=np.dtype("int8"))
dset = f.create_dataset(
dataset + "/video", (len(frame_numbers),), dtype=dtype
)
dset.attrs["format"] = format
dset.attrs["channels"] = self.channels
dset.attrs["height"] = self.height
dset.attrs["width"] = self.width
for i in range(len(frame_numbers)):
dset[i] = encode(frame_data[i])
else:
f.create_dataset(
dataset + "/video",
data=frame_data,
compression="gzip",
compression_opts=9,
)
if index_by_original:
f.create_dataset(dataset + "/frame_numbers", data=frame_numbers_data)
source_video_group = f.require_group(dataset + "/source_video")
source_video_dict = Video.cattr().unstructure(self)
source_video_group.attrs["json"] = json_dumps(source_video_dict)
return self.__class__(
backend=HDF5Video(
filename=path,
dataset=dataset + "/video",
input_format="channels_last",
convert_range=False,
)
)
