def load_masks(images):
"""
Return a list of boolean land masks.
Images must all be from the same station.
Arguments:
images (iterable): Image objects
"""
# All images must be from the same station (for now)
station = parse_image_path(images[0].path)['station']
pattern = re.compile(station + r'_[0-9]{8}_[0-9]{6}[^\/]*$')
is_station = [pattern.search(img.path) is not None for img in images[1:]]
assert all(is_station)
# Find all station svg with 'land' markup
imgsz = images[0].cam.imgsz
svg_paths = glob.glob(os.path.join(CG_PATH, 'svg', station + '_*.svg'))
markups = [glimpse.svg.parse_svg(path, imgsz=imgsz) for path in svg_paths]
land_index = np.where(['land' in markup for markup in markups])[0]
if len(land_index) == 0:
raise ValueError('No land masks found for station')
svg_paths = np.array(svg_paths)[land_index]
land_markups = np.array(markups)[land_index]
# Select svg files nearest to images, with preference within breaks
svg_datetimes = paths_to_datetimes(svg_paths)
svg_break_indices = np.array(
[_station_break_index(path) for path in svg_paths])
img_datetimes = [img.datetime for img in images]
distances = glimpse.helpers.pairwise_distance_datetimes(
img_datetimes, svg_datetimes)
nearest_index = []
for i, img in enumerate(images):
break_index = _station_break_index(img.path)
same_break = np.where(break_index == svg_break_indices)[0]
if same_break.size > 0:
i = same_break[np.argmin(distances[i][same_break])]
else:
raise ValueError('No mask found within motion breaks for image', i)
i = np.argmin(distances[i])
nearest_index.append(i)
nearest = np.unique(nearest_index)
# Make masks and expand per image without copying
masks = [None] * len(images)
image_sizes = np.array([img.cam.imgsz for img in images])
sizes = np.unique(image_sizes, axis=0)
for i in nearest:
polygons = land_markups[i]['land'].values()
is_nearest = nearest_index == i
for size in sizes:
scale = size / imgsz
rpolygons = [polygon * scale for polygon in polygons]
mask = glimpse.helpers.polygons_to_mask(rpolygons,
size=size).astype(np.uint8)
mask = sharedmem.copy(mask)
for j in np.where(is_nearest
& np.all(image_sizes == size, axis=1))[0]:
masks[j] = mask
return masks
def parse_image_path(path, sequence=False):
"""
Return metadata parsed from image path.
Arguments:
path (str): Image path or basename
sequence (bool): Whether to include sequence metadata (camera, service, ...)
"""
basename = glimpse.helpers.strip_path(path)
station, date_str, time_str = re.findall('^([^_]+)_([0-9]{8})_([0-9]{6})',
basename)[0]
capture_time = datetime.datetime.strptime(date_str + time_str,
'%Y%m%d%H%M%S')
results = dict(basename=basename,
station=station,
date_str=date_str,
time_str=time_str,
datetime=capture_time)
if sequence:
sequences = Sequences()
is_row = ((sequences.station == station) &
(sequences.first_time_utc <= capture_time) &
(sequences.last_time_utc >= capture_time))
rows = np.where(is_row)[0]
if len(rows) != 1:
raise ValueError(
'Image path has zero or multiple sequence matches: ' + path)
results = glimpse.helpers.merge_dicts(sequences.loc[rows[0]].to_dict(),
results)
return results
def _station_break_index(path):
"""
Return index of image in motion break sequence.
Arguments:
path (str): Image path
Returns:
int: Either 0 (original viewdir) or i (viewdir of break i + 1)
"""
stations = Stations()
ids = parse_image_path(path)
station = stations[ids['station']]
if 'breaks' not in station['properties']:
return 0
breaks = station['properties']['breaks']
if not breaks:
return 0
break_images = np.array([x['start'] for x in breaks])
idx = np.argsort(break_images)
i = np.where(break_images[idx] <= ids['basename'])[0]
if i.size > 0:
return idx[i[-1]] + 1
else:
return 0
def get_nearest_terminus(t):
"""
Return the terminus nearest a datetime.
"""
types = ('aerometric', 'arcticdem', 'ifsar', 'tandem', 'landsat-8',
'landsat-7', 'terrasar')
termini = [
f for f in Termini() if len(f['properties']['date']) == 10
and f['properties']['type'] in types
]
termini.sort(key=lambda x: (x['properties']['date'],
types.index(x['properties']['type'])))
datetimes = [
datetime.datetime.strptime(f['properties']['date'] + '22',
'%Y-%m-%d%H') for f in termini
]
dt = np.abs(np.array(datetimes) - t)
i = np.where(np.min(dt) == dt)[0][0]
return termini[i]['geometry']['coordinates']
os.path.join(rasters_path, 'nobservers.pkl'))
flotation = glimpse.helpers.read_pickle(
os.path.join(rasters_path, 'flotation.pkl'))
template = glimpse.Raster.read(os.path.join(rasters_path, 'template.tif'))
extension_x = glimpse.helpers.read_pickle(
os.path.join(rasters_path, 'extension_x.pkl'))
extension_y = glimpse.helpers.read_pickle(
os.path.join(rasters_path, 'extension_y.pkl'))
compression_x = glimpse.helpers.read_pickle(
os.path.join(rasters_path, 'compression_x.pkl'))
compression_y = glimpse.helpers.read_pickle(
os.path.join(rasters_path, 'compression_y.pkl'))
# Crop to coverage
raster = template.copy()
raster.Z = np.where((nobservers >= min_observers).any(axis=2), True, np.nan)
point_mask = raster.data_extent()
time_mask = (nobservers >= min_observers).any(axis=(0, 1))
indices = point_mask + (time_mask, )
raster.crop_to_data()
# Mask data
# dy, dx = np.gradient(nobservers, axis=(0, 1))
# few_obs = (nobservers < min_observers) | (dx != 0) | (dy != 0)
few_obs = (nobservers < min_observers)
# nobservers = nobservers.astype(float)
# nobservers[nobservers == 0] = np.nan
# few_obs |= (scipy.ndimage.minimum_filter(nobservers, size=(3, 3, 1)) < min_observers)
vx[few_obs] = np.nan
vy[few_obs] = np.nan
vz[few_obs] = np.nan
# _ = observers.pop(i)
# # Write to file
# glimpse.helpers.write_json(observers, path='observers.json', indent=4,
# flat_arrays=True)
# ---- Build Observers (rolling bins) ----
# Bins are shifted forward by a fixed amount.
# Choose bin sizes that evenly divide each range and least deviate from ideal
# min | dt - (n - 1) step - nominal_bin_dt |
# NOTE: Convert datetimes and timedeltas to float seconds to avoid ms rounding
step_dtS = step_dt.total_seconds()
nominal_bin_dtS = nominal_bin_dt.total_seconds()
coverageS = np.array([xi.timestamp() for xi in coverage.flat]).reshape(coverage.shape)
dtS = np.diff(coverageS, axis=1).ravel()
nbins = 1 + np.where(dtS > nominal_bin_dtS, (dtS - nominal_bin_dtS) / step_dtS, 0)
dsmaller = np.abs(dtS - (np.ceil(nbins) - 1) * step_dtS - nominal_bin_dtS)
dlarger = np.abs(dtS - (np.floor(nbins) - 1) * step_dtS - nominal_bin_dtS)
nbins = np.where(dsmaller < dlarger, np.ceil(nbins), np.floor(nbins)).astype(int)
bin_dtS = dtS - (nbins - 1) * step_dtS
# Compute Observer ranges
observer_ranges = []
for r, dt, n in zip(coverageS, bin_dtS, nbins):
starts = np.array([r[0] + ni * step_dtS for ni in range(n)])
ranges = np.column_stack((starts, starts + dt))
ranges = np.vectorize(datetime.datetime.fromtimestamp)(ranges)
observer_ranges.append(ranges)
# Build Observer image lists
observers = []
service_exif=True,
anchors=True,
viewdir=True,
viewdir_as_anchor=True)
matcher = glimpse.optimize.KeypointMatcher(images[:ends[tile]])
# De-anchor images in tile overlap
for img in matcher.images[starts[tile]:ends[tile - 1]]:
img.anchor = False
# Load matches for tile
read_matches(matcher, imgs=indices[starts[tile]:ends[tile]])
# Remove images with too few matches
# NOTE: Repeat until no additional images are below threshold
imgs = [None]
while len(imgs):
n = matcher.matches_per_image()
imgs = np.where(n < MIN_MATCHES)[0]
matcher.drop_images(imgs)
# Check for breaks in remaining matches
breaks = matcher.match_breaks()
if len(breaks):
raise ValueError('Match breaks at:', breaks)
# Check for an anchor image
is_anchor = np.array([img.anchor for img in matcher.images])
anchors = np.where(is_anchor)[0]
if not len(anchors):
raise ValueError('No anchor image present')
# Free up memory and convert matches to XY
matcher.filter_matches(clear_weights=True)
matcher.convert_matches(glimpse.optimize.RotationMatchesXY,
clear_uvs=True)
# Orient cameras
def load_images(station,
services,
use_exif=False,
service_exif=False,
anchors=False,
viewdir=True,
viewdir_as_anchor=False,
file_errors=True,
**kwargs):
"""
Return list of calibrated Image objects.
Any available station, camera, image, and viewdir calibrations are loaded
and images with image calibrations are marked as anchors.
Arguments:
station (str): Station identifier
services (iterable): Service identifiers
use_exif (bool): Whether to parse image datetimes from EXIF (slower)
rather than parsed from paths (faster)
service_exif (bool): Whether to extract EXIF from first image (faster)
or all images (slower) in service.
If `True`, `Image.datetime` is parsed from path.
Always `False` if `use_exif=True`.
anchors (bool): Whether to include anchor images even if
filtered out by `kwargs['snap']`
**kwargs: Arguments to `glimpse.helpers.select_datetimes()`
"""
if use_exif:
service_exif = False
# Sort services in time
if isinstance(services, str):
services = services,
services = np.sort(services)
# Parse datetimes of all candidate images
paths_service = [
glob.glob(
os.path.join(IMAGE_PATH, station, station + '_' + service,
'*.JPG')) for service in services
]
paths = np.hstack(paths_service)
basenames = [glimpse.helpers.strip_path(path) for path in paths]
if use_exif:
exifs = [glimpse.Exif(path) for path in paths]
datetimes = np.array([exif.datetime for exif in exifs])
else:
datetimes = paths_to_datetimes(basenames)
# Select images based on datetimes
indices = glimpse.helpers.select_datetimes(datetimes, **kwargs)
if anchors:
# Add anchors
# HACK: Ignore any <image>-<suffix>.json files
anchor_paths = glob.glob(
os.path.join(CG_PATH, 'images', station + '_*[0-9].json'))
anchor_basenames = [
glimpse.helpers.strip_path(path) for path in anchor_paths
]
if 'start' in kwargs or 'end' in kwargs:
# Filter by start, end
anchor_datetimes = np.asarray(paths_to_datetimes(anchor_basenames))
inrange = glimpse.helpers.select_datetimes(
anchor_datetimes,
**glimpse.helpers.merge_dicts(kwargs, dict(snap=None)))
anchor_basenames = np.asarray(anchor_basenames)[inrange]
anchor_indices = np.where(np.isin(basenames, anchor_basenames))[0]
indices = np.unique(np.hstack((indices, anchor_indices)))
service_breaks = np.hstack((0, np.cumsum([len(x) for x in paths_service])))
station_calibration = load_calibrations(station_estimate=station,
station=station,
merge=True,
file_errors=False)
images = []
for i, service in enumerate(services):
index = indices[(indices >= service_breaks[i])
& (indices < service_breaks[i + 1])]
if not index.size:
continue
service_calibration = glimpse.helpers.merge_dicts(
station_calibration,
load_calibrations(path=paths[index[0]],
camera=True,
merge=True,
file_errors=file_errors))
if service_exif:
exif = glimpse.Exif(paths[index[0]])
for j in index:
basename = basenames[j]
calibrations = load_calibrations(
image=basename,
viewdir=basename if viewdir else False,
station_estimate=station,
merge=False,
file_errors=False)
if calibrations['image']:
calibration = glimpse.helpers.merge_dicts(
service_calibration, calibrations['image'])
anchor = True
else:
calibration = glimpse.helpers.merge_dicts(
service_calibration,
dict(viewdir=calibrations['station_estimate']['viewdir']))
anchor = False
if viewdir and calibrations['viewdir']:
calibration = glimpse.helpers.merge_dicts(
calibration, calibrations['viewdir'])
if viewdir_as_anchor:
anchor = True
if use_exif:
exif = exifs[j]
elif not service_exif:
exif = None
if KEYPOINT_PATH:
keypoint_path = os.path.join(KEYPOINT_PATH, basename + '.pkl')
else:
keypoint_path = None
image = glimpse.Image(path=paths[j],
cam=calibration,
anchor=anchor,
exif=exif,
datetime=None if use_exif else datetimes[j],
keypoints_path=keypoint_path)
images.append(image)
return images