def test_int_id_to_binary():
"""Test conversion of integer id representation to binary array representation."""
bit_arr = int_id_to_binary(8)
assert np.all(bit_arr == np.array([0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
dtype=np.uint8))
bit_arr = int_id_to_binary(4095)
assert np.all(bit_arr == np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
dtype=np.uint8))
beeid_digits = 12
# test setting of single bits
expected_result = np.zeros((beeid_digits, beeid_digits), int)
np.fill_diagonal(expected_result, 1)
expected_result = np.fliplr(expected_result)
result = np.zeros_like(expected_result)
for i in range(0, beeid_digits):
result[i] = int_id_to_binary(2**i, nb_bits=beeid_digits)
assert np.array_equal(result, expected_result)
# correct conversion of all integers in range
for i in range(0, 2**beeid_digits):
bit_array = int_id_to_binary(i, nb_bits=beeid_digits)
assert beeid_digits == len(bit_array)
assert i == sum([m * 2**n for (n, m) in enumerate(bit_array[::-1])])
with pytest.raises(Exception) as exception_information: # to big value
bit_arr = int_id_to_binary(8096)
assert 'overflows' in str(exception_information.value)
def score_ids_best_fit(ids1, ids2, length=12):
"""Compares two lists of ids by choosing the pair with the best score.
.. deprecated:: September 2016
This scoring is used with data from the *old* Computer Vision Pipeline.
Arguments:
ids1 (:obj:`list` of int): Iterable with ids as integers as base ids
ids2 (:obj:`list` of int): Iterable with ids as integers to compare base to
Keyword Arguments:
length (Optional int): number of bits in the bit array used to compare ids
Returns:
float: Uses Hamming distance of best matching pair
"""
assert len(ids1) > 0
assert len(ids2) > 0
ids1, ids2 = set(ids1), set(ids2)
# best case: we already have the same ids in both sets
if ids1 & ids2:
return 0
best_score = float("inf")
for id1 in ids1:
id1 = int_id_to_binary(id1, nb_bits=length)
for id2 in ids2:
id2 = int_id_to_binary(id2, nb_bits=length)
test_score = hamming(id1, id2)
if test_score < best_score:
best_score = test_score
return best_score
def test_int_id_to_binary(int_bin_mapping):
"""Test conversion of integer id representation to binary array representation."""
for int_repr, bin_repr in int_bin_mapping:
assert np.all(int_id_to_binary(int_repr) == bin_repr)
with pytest.raises(Exception) as exception_information: # to big value
int_id_to_binary(8096)
assert 'overflows' in str(exception_information.value)
def test_int_id_to_binary(int_bin_mapping):
"""Test conversion of integer id representation to binary array representation."""
for int_repr, bin_repr in int_bin_mapping:
assert np.all(int_id_to_binary(int_repr) == bin_repr)
with pytest.raises(Exception) as exception_information: # to big value
int_id_to_binary(8096)
assert 'overflows' in str(exception_information.value)
def score_ids_and_orientation(id_orientation_tuple1,
id_orientation_tuple2,
length=12,
range_bonus_orientation=30):
"""Compares lists of ids by choosing the pair with the best score and considering orientation.
The bonus is equal to the negative score of one non matching bit.
.. deprecated:: September 2016
This scoring is used with data from the *old* Computer Vision Pipeline.
Arguments:
id_orientation_tuple1 (tuple): (Iterable with ids, Iterable with orientations)
id_orientation_tuple2 (tuple): (Iterable with ids, Iterable with orientations)
Keyword Arguments:
length (int): number of bits in the bit array used to compare ids
range_bonus_orientation (float): range in degrees, so that two orientations get a bonus
Returns:
float: Uses Hamming distance of best matching pair with bonus for same orientation
"""
ids1, orientations1 = id_orientation_tuple1
ids2, orientations2 = id_orientation_tuple2
assert len(ids1) > 0
assert len(ids2) > 0
assert len(ids1) == len(orientations1)
assert len(ids2) == len(orientations2)
# best case: we already have the same ids in both sets
if set(ids1) & set(ids2):
return 0
best_score = float("inf")
for id1, or1 in zip(ids1, orientations1):
id1 = int_id_to_binary(id1, nb_bits=length)
for id2, or2 in zip(ids2, orientations2):
id2 = int_id_to_binary(id2, nb_bits=length)
# bonus for orientation
orientation_score = 0.0
if distance_orientations(or1, or2) <= range_bonus_orientation:
orientation_score = -1. / length
test_score = hamming(id1, id2) + orientation_score
if test_score < best_score:
best_score = test_score
if best_score <= 0:
return best_score
return best_score
def run(gt_file,
videos,
images,
visualize_debug,
output,
fix_utc_2014,
nb_bits=12):
"""
Converts bb_binary ground truth Cap'n Proto files to hdf5 files and
extracts the corresponding rois from videos or images.
"""
def get_filenames(f):
if f is None:
return []
else:
return [line.rstrip('\n') for line in f.readlines()]
gen_factory = FrameGeneratorFactory(get_filenames(videos),
get_filenames(images))
if os.path.exists(output):
os.remove(output)
distribution = DistributionCollection([('bits', Bernoulli(), nb_bits)])
dset = DistributionHDF5Dataset(output, distribution)
camIdxs = []
periods = []
for fname in gt_file:
fc = load_frame_container(fname)
camIdx, start_dt, end_dt = parse_video_fname(fname)
if fix_utc_2014 and start_dt.year == 2014:
start_dt -= timedelta(hours=2)
gt_frames = []
gen = gen_factory.get_generator(camIdx, start_dt)
for frame, (video_frame, video_filename) in zip(fc.frames, gen):
gt = {}
np_frame = convert_frame_to_numpy(frame)
rois, mask, positions = extract_gt_rois(np_frame, video_frame,
start_dt)
for name in np_frame.dtype.names:
gt[name] = np_frame[name][mask]
bits = [int_id_to_binary(id)[::-1] for id in gt["decodedId"]]
gt["bits"] = 2 * np.array(bits, dtype=np.float) - 1
gt["tags"] = 2 * (rois / 255.).astype(np.float16) - 1
gt['filename'] = os.path.basename(video_filename)
gt['camIdx'] = camIdx
gt_frames.append(gt)
print('.', end='', flush=True)
print()
gt_period = GTPeriod(camIdx, start_dt, end_dt, fname, gt_frames)
periods.append(
[int(gt_period.start.timestamp()),
int(gt_period.end.timestamp())])
camIdxs.append(gt_period.camIdx)
append_gt_to_hdf5(gt_period, dset)
dset.attrs['periods'] = np.array(periods)
dset.attrs['camIdxs'] = np.array(camIdxs)
visualize_detection_tiles(dset, os.path.splitext(output)[0])
dset.close()
def score_ids_best_fit_rotating(ids1, ids2, rotation_penalty=1, length=12):
"""Compares two lists of ids by choosing the pair best score by rotating the bits.
.. deprecated:: September 2016
This scoring is used with data from the *old* Computer Vision Pipeline.
Arguments:
ids1 (:obj:`list` of int): Iterable with ids as integers as base ids
ids2 (:obj:`list` of int): Iterable with ids as integers to compare base to
Keyword Arguments:
rotation_penalty (Optional float): the penalty that is added for a rotation
of 1 to the left or right
length (Optional int): number of bits in the bit array used to compare ids
Returns:
float: Uses Hamming distance of best matching (rotated) pair
"""
assert len(ids1) > 0
assert len(ids2) > 0
# best case: we already have the same ids in both sets
if set(ids1) & set(ids2):
return 0
rotation_penalty = float(rotation_penalty) / length
best_score = float("inf")
for id1 in ids1:
id1 = int_id_to_binary(id1, nb_bits=length)
for id2 in ids2:
id2 = int_id_to_binary(id2, nb_bits=length)
# rotate left and right
for direction in [-1, 1]:
for i in range(int(math.ceil(float(length / 2))) + 1):
if i * rotation_penalty >= best_score:
break
test_score = hamming(id1, np.roll(
id2, direction * i)) + i * rotation_penalty
if test_score < best_score:
best_score = test_score
if best_score <= 0:
return best_score
return best_score
def test_binary_id_to_int(int_bin_mapping):
"""Test conversion of binary array representation to interger id."""
for int_repr, bin_repr in int_bin_mapping:
assert binary_id_to_int(bin_repr) == int_repr
# test with floats
for int_repr, bin_repr in int_bin_mapping:
assert binary_id_to_int(bin_repr / 2) == int_repr
# change endianess
for int_repr, bin_repr in int_bin_mapping:
bin_repr = bin_repr[::-1]
assert binary_id_to_int(bin_repr / 2, endian='little') == int_repr
# different threshold
bit_array = np.arange(0, 1.2, 0.1)
assert len(bit_array) == 12
assert np.sum(int_id_to_binary(binary_id_to_int(bit_array))) == 12 - 5
assert np.sum(int_id_to_binary(binary_id_to_int(bit_array, threshold=0))) == 12
assert np.sum(int_id_to_binary(binary_id_to_int(bit_array, threshold=1))) == 2
def test_bit_array_to_int_v():
"""Tests the conversion of bit arrays to integers (vectorized)."""
expected_ids, detections = [], []
for i in range(2**12):
expected_ids.append(i)
detections.append(make_detection(beeid=int_id_to_binary(i)[::-1] / 2))
calculated_ids = bit_array_to_int_v(detections)
assert len(expected_ids) == len(calculated_ids)
assert set(expected_ids) == set(calculated_ids)
with pytest.raises(AssertionError):
bit_array_to_int_v([make_detection(beeid=[1] * 13)])
def test_binary_id_to_int(int_bin_mapping):
"""Test conversion of binary array representation to interger id."""
for int_repr, bin_repr in int_bin_mapping:
assert binary_id_to_int(bin_repr) == int_repr
# test with floats
for int_repr, bin_repr in int_bin_mapping:
assert binary_id_to_int(bin_repr / 2) == int_repr
# change endianess
for int_repr, bin_repr in int_bin_mapping:
bin_repr = bin_repr[::-1]
assert binary_id_to_int(bin_repr / 2, endian='little') == int_repr
# different threshold
bit_array = np.arange(0, 1.2, 0.1)
assert len(bit_array) == 12
assert np.sum(int_id_to_binary(binary_id_to_int(bit_array))) == 12 - 5
assert np.sum(int_id_to_binary(binary_id_to_int(bit_array,
threshold=0))) == 12
assert np.sum(int_id_to_binary(binary_id_to_int(bit_array,
threshold=1))) == 2
def run(bb_gt_files, video_dir, image_dir, visualize_debug, force, output):
"""
Converts bb_binary ground truth Cap'n Proto files to hdf5 files and
extracts the corresponding rois from videos or images.
"""
gen_factory = FrameGeneratorFactory(video_dir, image_dir)
if force and os.path.exists(output):
os.remove(output)
dset = HDF5Dataset(output)
camIdxs = []
periods = []
for fname in bb_gt_files:
fc = load_frame_container(fname)
camIdx, start_dt, end_dt = parse_video_fname(fname)
basename = os.path.basename(fname)
gt_frames = []
print(basename)
gen = gen_factory.get_generator(camIdx, start_dt)
first = True
for frame, (video_frame, video_filename) in zip(fc.frames, gen):
gt = {}
np_frame = convert_frame_to_numpy(frame)
rois, mask, positions = extract_gt_rois(np_frame, video_frame,
start_dt)
for name in np_frame.dtype.names:
gt[name] = np_frame[name][mask]
gt["bits"] = np.array(
[int_id_to_binary(id)[::-1] for id in gt["decodedId"]])
gt["tags"] = rois
gt['filename'] = os.path.basename(video_filename)
gt_frames.append(gt)
if first and visualize_debug:
visualize_detections(gt, positions, video_frame)
first = False
print('.', end='')
gt_period = GTPeriod(camIdx, start_dt, end_dt, fname, gt_frames)
periods.append(
[int(gt_period.start.timestamp()),
int(gt_period.end.timestamp())])
camIdxs.append(gt_period.camIdx)
append_gt_to_hdf5(gt_period, dset)
dset.attrs['periods'] = np.array(periods)
dset.attrs['camIdxs'] = np.array(camIdxs)
dset.close()
def run(bb_gt_files, video_dir, image_dir, visualize_debug, force, output):
"""
Converts bb_binary ground truth Cap'n Proto files to hdf5 files and
extracts the corresponding rois from videos or images.
"""
gen_factory = FrameGeneratorFactory(video_dir, image_dir)
if force and os.path.exists(output):
os.remove(output)
dset = HDF5Dataset(output)
camIdxs = []
periods = []
for fname in bb_gt_files:
fc = load_frame_container(fname)
camIdx, start_dt, end_dt = parse_video_fname(fname)
basename = os.path.basename(fname)
gt_frames = []
print(basename)
gen = gen_factory.get_generator(camIdx, start_dt)
first = True
for frame, (video_frame, video_filename) in zip(fc.frames, gen):
gt = {}
np_frame = convert_frame_to_numpy(frame)
rois, mask, positions = extract_gt_rois(np_frame, video_frame, start_dt)
for name in np_frame.dtype.names:
gt[name] = np_frame[name][mask]
gt["bits"] = np.array([int_id_to_binary(id)[::-1] for id in gt["decodedId"]])
gt["tags"] = rois
gt['filename'] = os.path.basename(video_filename)
gt_frames.append(gt)
if first and visualize_debug:
visualize_detections(gt, positions, video_frame)
first = False
print('.', end='')
gt_period = GTPeriod(camIdx, start_dt, end_dt, fname, gt_frames)
periods.append([int(gt_period.start.timestamp()), int(gt_period.end.timestamp())])
camIdxs.append(gt_period.camIdx)
append_gt_to_hdf5(gt_period, dset)
dset.attrs['periods'] = np.array(periods)
dset.attrs['camIdxs'] = np.array(camIdxs)
dset.close()