def test_parse_image_fname(image):
"""Tests the extraction of camera, date and time information from image filenames."""
if image['format'] == 'arbitrary':
with pytest.raises(Exception):
camIdx, ts = parse_image_fname(image['name'], format=image['format'])
return
camIdx, ts = parse_image_fname(image['name'], format=image['format'])
assert camIdx == image['cam']
assert ts == image['dt']
def test_parse_image_fname(image):
"""Tests the extraction of camera, date and time information from image filenames."""
if image['format'] == 'arbitrary':
with pytest.raises(Exception):
camIdx, ts = parse_image_fname(image['name'],
format=image['format'])
return
camIdx, ts = parse_image_fname(image['name'], format=image['format'])
assert camIdx == image['cam']
assert ts == image['dt']
def __init__(self, video_dir, image_dir):
self.videos = get_files(video_dir)
self.image_dirs = get_subdirs(image_dir)
self.videos_dict = {parse_video_fname(video)[:2]: video for video in self.videos}
self.image_dirs_dict = {}
for dir in self.image_dirs:
first_frame = sorted(get_files(dir))[0]
parsed = parse_image_fname(first_frame)
self.image_dirs_dict[parsed] = dir
def __init__(self, video_dir, image_dir):
self.videos = get_files(video_dir)
self.image_dirs = get_subdirs(image_dir)
self.videos_dict = {
parse_video_fname(video)[:2]: video
for video in self.videos
}
self.image_dirs_dict = {}
for dir in self.image_dirs:
first_frame = sorted(get_files(dir))[0]
parsed = parse_image_fname(first_frame)
self.image_dirs_dict[parsed] = dir
def __init__(self, video_files, image_dirs):
self.videos = video_files
self.image_dirs = image_dirs
self.sources = {
parse_video_fname(video)[:2]: ('video', video)
for video in self.videos
}
for dir in self.image_dirs:
first_frame = sorted(get_files(dir))[0]
parsed = parse_image_fname(first_frame)
self.sources[parsed] = ('image', dir)
self.sources = {(cam, drop_microseconds(ts)): val
for (cam, ts), val in self.sources.items()}
def get_timestamps(path_video, ts_format, path_filelists):
# In season 2014 and 2015, the timestamps for each video were stored in a seperate location
# from the videos themselves. From 2016 onwards a text file with the timestamps of all frames
# is stored next to the videos with the same filename (apart from the extension).
if ts_format == '2016':
assert path_filelists is None
txt_path = path_video.replace('mkv', 'txt')
fnames = open(txt_path, 'r').readlines()
return [
parse_image_fname(fn, format='iso')[1].timestamp() for fn in fnames
]
elif ts_format in ('2014', '2015'):
assert path_filelists is not None
return get_seperate_timestamps(path_video, ts_format, path_filelists)
else:
assert False, 'Unknown timestamp format'
def run(image_path_file, out, force):
cache = {}
if force and os.path.exists(out):
print("Removing {}.".format(out))
os.remove(out)
def add_to_cache(**kwargs):
for name, arr in kwargs.items():
if name not in cache:
cache[name] = [arr]
else:
cache[name].append(arr)
def flush_cache():
cache_concat = {n: np.concatenate(arrs) for n, arrs in cache.items()}
nb_samples = len(next(iter(cache_concat.values())))
permutation = np.random.permutation(nb_samples)
cache_shuffled = {n: arrs[permutation]
for n, arrs in cache_concat.items()}
dset.append(**cache_shuffled)
cache.clear()
roi_size = 96
image_fnames = [n.rstrip('\n') for n in image_path_file.readlines()]
pipeline = Pipeline([Filename],
[Image, LocalizerPositions],
**get_auto_config())
dset = HDF5Dataset(out)
bar = progressbar.ProgressBar(max_value=len(image_fnames))
for i, image_fname in enumerate(bar(image_fnames)):
try:
results = pipeline([image_fname])
rois, mask = Localizer.extract_rois(results[LocalizerPositions],
results[Image], roi_size)
except Exception as e:
print(e)
continue
nb_detections = np.sum(mask)
camIdx, dt = parse_image_fname(image_fname)
season = np.array([dt.year] * nb_detections, dtype=np.uint16)
timestamp = np.array([dt.timestamp()] * nb_detections, dtype=np.float64)
add_to_cache(rois=rois, season=season, timestamp=timestamp)
if i % 50 == 0 and i != 0:
flush_cache()
flush_cache()
def get_timestamps(path_video, ts_format, path_filelists):
# In season 2014 and 2015, the timestamps for each video were stored in a
# seperate location from the videos themselves. From 2016 onwards a text
# file with the timestamps of all frames is stored next to the videos with
# the same filename (apart from the extension).
if ts_format == "2016":
assert path_filelists is None
file_type = path_video.split(".")[-1]
txt_path = path_video[: -len(file_type)] + "txt"
with open(txt_path) as f:
fnames = f.read().splitlines()
return [parse_image_fname(fn, format="iso")[1].timestamp() for fn in fnames]
elif ts_format in ("2014", "2015"):
assert path_filelists is not None
return get_seperate_timestamps(path_video, ts_format, path_filelists)
else:
assert False, "Unknown timestamp format"
def get_seperate_timestamps(path_video, ts_format, path_filelists):
def get_flist_name(dt_utc):
fmt = '%Y%m%d'
dt = dt_utc.astimezone(get_timezone())
if ts_format == '2014':
return dt.strftime(fmt) + '.txt'
elif ts_format == '2015':
return os.path.join(dt.strftime(fmt), 'images.txt')
else:
assert (False)
def find_file(name, path):
for root, dirs, files in os.walk(path):
if name in [
os.path.join(os.path.basename(root), f) for f in files
]:
return os.path.join(path, name)
return None
def next_day(dt):
return dt + timedelta(days=1)
fname_video = os.path.basename(path_video)
cam, from_dt, to_dt = parse_video_fname(fname_video)
# due to a bug in the data aquistion software we have to check in the
# filename list of the next day as well
timestamps = [from_dt, to_dt, next_day(from_dt), next_day(to_dt)]
txt_files = set([get_flist_name(dt) for dt in timestamps])
txt_paths = [find_file(f, path_filelists) for f in txt_files]
txt_paths = [path for path in txt_paths if path is not None]
assert len(txt_paths) > 0
image_fnames = list(
chain.from_iterable(
[open(path, 'r').readlines() for path in txt_paths]))
first_fname = fname_video.split('_TO_')[0] + '.jpeg\n'
second_fname = fname_video.split('_TO_')[1].split('.mkv')[0] + '.jpeg\n'
image_fnames.sort()
fnames = image_fnames[image_fnames.
index(first_fname):image_fnames.index(second_fname) +
1]
return [
parse_image_fname(fn, format='beesbook')[1].timestamp()
for fn in fnames
]
def call(self, fname):
image = imread(fname)
camIdx, dt = parse_image_fname(fname)
return image, dt.timestamp(), camIdx
def call(self, fname):
image = imread(fname)
camIdx, dt = parse_image_fname(fname)
return image, dt.timestamp(), camIdx