def verify_geospatial_bounds(self, ds):
"""Checks that the geospatial bounds is well formed OGC WKT"""
var = getattr(ds, "geospatial_bounds", None)
check = var is not None
if not check:
return ratable_result(
False,
"Global Attributes", # grouped with Globals
["geospatial_bounds not present"],
)
try:
# TODO: verify that WKT is valid given CRS (defaults to EPSG:4326
# in ACDD.
from_wkt(ds.geospatial_bounds)
except AttributeError:
return ratable_result(
False,
"Global Attributes", # grouped with Globals
[
(
"Could not parse WKT from geospatial_bounds,"
' possible bad value: "{}"'.format(ds.geospatial_bounds)
)
],
variable_name="geospatial_bounds",
)
# parsed OK
else:
return ratable_result(True, "Global Attributes", tuple())
def test_speed(self):
def parse_wkt(text):
ast = parser.parse(text, rule_name='well_known_text_representation', semantics=semantics)
if isinstance(ast, dict):
return ast
else:
raise NotImplementedError
from time import time
arr = [
#tri0, tin0, phs1, phs0,
#empty_gc0, empty_gc1, empty_gc2,
#empty_mpoly0, empty_mpoly1, empty_poly0, empty_poly1,
#empty_mls1, empty_mls0, empty_ls1, empty_ls0, empty_mp0,
#empty_mp1, empty_p0, empty_p1, mpoly_empty,
poly0, poly1, poly2, poly3,
poly4, point_m0, point_m1, p0, p_zm, p_z, ms0, ms1,
mpoly0, mpoly1, mpoly2, mpoly3, mp0, mp1, mp2, mp3, mp4,
mls0, mls1, mls2, mls3, mls4, mls5,
#mc0, mc1, mc2, mc3,
ls0, ls1, ls2, ls3, gc0, gc1, gc2, gc3, gc4, gc5, gc6,
#cs0, cs1,
#cs2, cs3, cs4, cs5, cp0, cp1, cc0, cc1
]
parser = WktParser(parseinfo=False)
semantics=Semantics()
pwktf = 0
pgif = 0
for g in arr:
try:
from_wkt(g)
except:
pwktf +=1
try:
pygeoif.from_wkt(g)
except:
pgif +=1
#print('Pygeoif Failures:', pgif)
#print('ParseWKT failures:', pwktf)
num_loops = 10
t0 = time()
for i in range(1, num_loops):
for g in arr:
try:
parse_wkt(g)
except:
pwktf +=1
pwktt = time() - t0
t0 = time()
for i in range(1, num_loops):
for g in arr:
try:
pygeoif.from_wkt(g)
except:
pgif +=1
pygit = time() - t0
#print('Pygeoif time:', pygit)
#print('ParseWKT time:', pwktt)
#print('wkt/re:', pwktt/pygit)
self.assertTrue(pwktt/pygit > 10)
def create_mask_junctions(lines, thickness=16):
mask = np.zeros((1300, 1300))
point_map = {}
for line in lines:
wkt_pix = line["wkt_pix"]
if "EMPTY" not in wkt_pix:
line = wkt_pix
points = pygeoif.from_wkt(line).coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
point_map[ds_point(pt1)] = point_map.get(ds_point(pt1), 0) + 1
point_map[ds_point(pt2)] = point_map.get(ds_point(pt2), 0) + 1
for line in lines:
wkt_pix = line["wkt_pix"]
if "EMPTY" not in wkt_pix:
line = wkt_pix
points = pygeoif.from_wkt(line).coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
pt2_ori = pt2
if point_map[ds_point(pt1)] < 3 and point_map[ds_point(
pt2)] < 3:
continue
if point_map[ds_point(pt1)] >= 3:
distance = euclidean(pt1, pt2)
if distance > 32:
frac = 32 / distance
xc = pt1[0] + (pt2[0] - pt1[0]) * frac
yc = pt1[1] + (pt2[1] - pt1[1]) * frac
pt2 = (int(xc), int(yc))
cv2.line(mask, pt1, pt2, (1, ), thickness=thickness)
pt2 = pt2_ori
if point_map[ds_point(pt2)] >= 3:
tmp = pt1
pt1 = pt2
pt2 = tmp
distance = euclidean(pt1, pt2)
if distance > 32:
frac = 32 / distance
xc = pt1[0] + (pt2[0] - pt1[0]) * frac
yc = pt1[1] + (pt2[1] - pt1[1]) * frac
pt2 = (int(xc), int(yc))
cv2.line(mask, pt1, pt2, (1, ), thickness=thickness)
return mask * 255
def increasing_area(p, results='start', step=0):
# check if square contains point
# if not recursively increase the size
if results != 'start' and results or step == 25:
return results[0] if results else None
print(p, step)
query = select([Areas.center
]).where(func.ST_Contains(Areas.aoi, p))
#print(str(query.compile()))
results = areasOps.exec_func_query(query, multi=True)
lookup = from_wkt(p).__geo_interface__['coordinates']
stepping = lookup
for r in range(100):
s = step - 4
if mapping.get(str(s), None):
if step % 2 == 0:
stepping = (lookup[mapping.get('2')[0]] +
mapping.get('2')[1], lookup[1])
else:
stepping = (lookup[0], lookup[mapping.get('3')[0]] +
mapping.get('3')[1])
step += 1
new_point = spatial.shape_geometry(stepping[0], stepping[1])
return increasing_area(new_point, results, step)
def verify_geospatial_bounds(self, ds):
"""Checks that the geospatial bounds is well formed OGC WKT"""
var = getattr(ds, 'geospatial_bounds', None)
check = var is not None
if not check:
return ratable_result(False, 'geospatial_bounds',
["Attr geospatial_bounds not present"])
try:
# TODO: verify that WKT is valid given CRS (defaults to EPSG:4326
# in ACDD.
from_wkt(ds.geospatial_bounds)
except AttributeError:
return ratable_result(
False, 'geospatial_bounds',
['Could not parse WKT, possible bad value for WKT'])
# parsed OK
else:
return ratable_result(True, 'geospatial_bounds', tuple())
def verify_geospatial_bounds(self, ds):
"""Checks that the geospatial bounds is well formed OGC WKT"""
var = getattr(ds, 'geospatial_bounds', None)
check = var is not None
if not check:
return ratable_result(False,
"Global Attributes", # grouped with Globals
["geospatial_bounds not present"])
try:
# TODO: verify that WKT is valid given CRS (defaults to EPSG:4326
# in ACDD.
from_wkt(ds.geospatial_bounds)
except AttributeError:
return ratable_result(False,
"Global Attributes", # grouped with Globals
['Could not parse WKT, possible bad value for WKT'])
# parsed OK
else:
return ratable_result(True, "Global Attributes", tuple())
def create_mask(lines, thickness=16):
mask = np.zeros((1300, 1300))
for line in lines:
wkt_pix = line["wkt_pix"]
if "EMPTY" not in wkt_pix:
line = wkt_pix
points = pygeoif.from_wkt(line).coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
cv2.line(mask, pt1, pt2, (1, ), thickness=thickness)
return mask * 255
def _create_mask(self, image_id: str, image: np.ndarray) -> np.ndarray:
h, w, _ = image.shape
mask = np.zeros(shape=(h, w), dtype=np.uint8)
for line in self._roads_data[image_id.replace('_PS-RGB', '').replace('SN3_roads_train_', '')]:
if "LINESTRING EMPTY" == line:
continue
points = pygeoif.from_wkt(line).coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
cv2.line(mask, pt1, pt2, 255, thickness=self._thickness)
return mask
def __init__(self, geometry):
try:
if any(geometry.find(e) != -1 for e in self.elements):
self.element = [e for e in self.elements if geometry.find(e) != -1][0]
else:
raise ValueError('Controller: geometry has to be '
'a EWKT POINT or POLYGON')
except Exception as e:
raise ValueError('The value passed at Controller is not in the right format: {}'.format(
str(e)
)
)
self.geometry = geometry # EWKT string
self.geo_object = from_wkt(self.geometry) # pygeoif object
# #todo: implement descriptors
self.results_proxy = None # variable to store the resulting areas
self.geojson = None # variable to store the resulting geojson
def __init__(self, geometry):
try:
if any(geometry.find(e) != -1 for e in self.elements):
self.element = [
e for e in self.elements if geometry.find(e) != -1
][0]
else:
raise ValueError('Controller: geometry has to be '
'a EWKT POINT or POLYGON')
except Exception as e:
raise ValueError(
'The value passed at Controller is not in the right format: {}'
.format(str(e)))
self.geometry = geometry # EWKT string
self.geo_object = from_wkt(self.geometry) # pygeoif object
# #todo: implement descriptors
self.results_proxy = None # variable to store the resulting areas
self.geojson = None # variable to store the resulting geojson
def increasing_area(p, results='start', step=0):
# check if square contains point
# if not recursively increase the size
if results != 'start' and results or step == 25:
return results[0] if results else None
print(p, step)
query = select([Areas.center]).where(func.ST_Contains(Areas.aoi, p))
#print(str(query.compile()))
results = areasOps.exec_func_query(query, multi=True)
lookup = from_wkt(p).__geo_interface__['coordinates']
stepping = lookup
for r in range(100):
s = step - 4
if mapping.get(str(s), None):
if step % 2 == 0:
stepping = (lookup[mapping.get('2')[0]] + mapping.get('2')[1], lookup[1])
else:
stepping = (lookup[0], lookup[mapping.get('3')[0]] + mapping.get('3')[1])
step += 1
new_point = spatial.shape_geometry(stepping[0], stepping[1])
return increasing_area(new_point, results, step)
def create_speed_mask(lines, thickness=16):
max_speed = 35 # mps
mask = np.zeros((1300, 1300))
for line in lines:
wkt_pix = line["wkt_pix"]
length_m = line["length_m"]
travel_time_s = line["travel_time_s"]
if "EMPTY" not in wkt_pix:
speed = 9. if travel_time_s == 0 else length_m / travel_time_s
speed_normalized = int(255 * speed / max_speed)
line = wkt_pix
wkt = pygeoif.from_wkt(line)
points = wkt.coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
cv2.line(mask,
pt1,
pt2, (speed_normalized, ),
thickness=thickness)
return mask
def _get_batches_of_transformed_samples(self, index_array):
batch_x = []
batch_y = []
for batch_index, image_index in enumerate(index_array):
city, id = self.image_ids[image_index]
for data_dir in self.data_dirs:
city_dir_name = data_dir.split("/")[-1]
if city in data_dir:
img_name = self.image_name_template.format(id=id)
if self.clahe:
data_dir = os.path.join(self.wdata_dir, city_dir_name)
path = os.path.join(data_dir, img_name)
else:
path = os.path.join(data_dir, img_name)
break
arr = tifffile.imread(path)
image = np.stack([
arr[..., 4], arr[..., 2], arr[..., 1], arr[..., 0],
arr[..., 3], arr[..., 5], arr[..., 6], arr[..., 7]
],
axis=-1)
if self.stretch_and_mean:
image = stretch_8bit(image) * 255
if self.ohe_city:
ohe_city = np.zeros((image.shape[0], image.shape[1], 4),
dtype="float32")
ohe_city[..., cities.index(city)] = 2047
image = np.concatenate([image, ohe_city], axis=-1)
image = np.array(image, dtype="float32")
lines = self.masks_dict[id]
mask = np.zeros((image.shape[0], image.shape[1], 1))
# lines in wkt format, pygeoif
if os.path.exists("masks/" + id + ".png"):
mask = img_to_array(
load_img("masks/" + id + ".png", grayscale=True)) / 255.
else:
mask = np.zeros((image.shape[0], image.shape[1], 1))
# lines in wkt format, pygeoif
for line in lines:
if "LINESTRING EMPTY" == line:
continue
points = pygeoif.from_wkt(line).coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
cv2.line(mask,
pt1,
pt2, (1, ),
thickness=self.thickness)
cv2.imwrite("masks/" + id + ".png", mask * 255)
ori_height = image.shape[0]
ori_width = image.shape[1]
mask = self.transform_mask(mask, image)
if self.random_transformer is not None:
image, mask = self.random_transformer.random_transform(
image, mask)
if self.stretch_and_mean:
mean_band = mean_bands[city]
for band in range(len(mean_band)):
image[..., band] -= mean_band[band]
if self.crop_shape is not None:
crops = 0
tries = 0
while crops < self.crops_per_image:
tries += 1
if self.random_transformer is None:
y_start = (ori_height - self.crop_shape[0]) // 2
x_start = (ori_width - self.crop_shape[1]) // 2
else:
y_start = random.randint(
0, ori_height - self.crop_shape[0] - 1)
x_start = random.randint(
0, ori_width - self.crop_shape[1] - 1)
y_end = y_start + self.crop_shape[0]
x_end = x_start + self.crop_shape[1]
crop_image = image[y_start:y_end, x_start:x_end, :]
crop_mask = mask[y_start:y_end, x_start:x_end, :]
if self.random_transformer is None:
batch_x.append(crop_image)
batch_y.append(crop_mask)
crops += 1
elif np.count_nonzero(crop_image) > 100 or tries > 20:
batch_x.append(crop_image)
batch_y.append(crop_mask)
crops += 1
else:
batch_x.append(image)
batch_y.append(mask)
batch_x = np.array(batch_x, dtype="float32")
batch_y = np.array(batch_y, dtype="float32")
if self.preprocessing_function == 'caffe':
batch_x_rgb = batch_x[..., :3]
batch_x_bgr = batch_x_rgb[..., ::-1]
batch_x[..., :3] = batch_x_bgr
if not self.stretch_and_mean:
batch_x = batch_x / 8. - 127.5
else:
if self.stretch_and_mean:
batch_x = batch_x / 255
else:
batch_x = batch_x / 1024. - 1
return self.transform_batch_x(batch_x), self.transform_batch_y(batch_y)
def geom_to_list_point2d(self, geom):
geom_wkt = self.execute_function(geom_as_text(geom))
coords = from_wkt(geom_wkt).coords
return [Point2D(c[0], c[1]) for c in coords]
def geom_to_point2d(self, geom):
geom_wkt = self.execute_function(geom_as_text(geom))
(x, y, z) = from_wkt(geom_wkt).coords[0]
return Point2D(x, y)
def _get_batches_of_transformed_samples(self, index_array):
batch_x = []
batch_y = []
for batch_index, image_index in enumerate(index_array):
city, id = self.image_ids[image_index]
for data_dir in self.data_dirs:
city_dir_name = data_dir.split("/")[-1]
if city in data_dir:
img_name = self.image_name_template.format(id=id)
if self.clahe:
data_dir = os.path.join(self.wdata_dir, city_dir_name)
path = os.path.join(data_dir, img_name)
else:
path = os.path.join(data_dir, img_name)
break
arr = tifffile.imread(path)
image = np.stack([arr[..., 4], arr[..., 2], arr[..., 1], arr[..., 0], arr[..., 3], arr[..., 5], arr[..., 6], arr[..., 7]], axis=-1)
if self.stretch_and_mean:
image = stretch_8bit(image) * 255
if self.ohe_city:
ohe_city = np.zeros((image.shape[0], image.shape[1], 4), dtype="float32")
ohe_city[..., cities.index(city)] = 2047
image = np.concatenate([image, ohe_city], axis=-1)
image = np.array(image, dtype="float32")
lines = self.masks_dict[id]
mask = np.zeros((image.shape[0], image.shape[1], 1))
# lines in wkt format, pygeoif
if os.path.exists("masks/" + id + ".png"):
mask = img_to_array(load_img("masks/" + id + ".png", grayscale=True)) / 255.
else:
mask = np.zeros((image.shape[0], image.shape[1], 1))
# lines in wkt format, pygeoif
for line in lines:
if "LINESTRING EMPTY" == line:
continue
points = pygeoif.from_wkt(line).coords
for i in range(1, len(points)):
pt1 = (int(points[i - 1][0]), int(points[i - 1][1]))
pt2 = (int(points[i][0]), int(points[i][1]))
cv2.line(mask, pt1, pt2, (1,), thickness=self.thickness)
cv2.imwrite("masks/" + id + ".png", mask * 255)
ori_height = image.shape[0]
ori_width = image.shape[1]
mask = self.transform_mask(mask, image)
if self.random_transformer is not None:
image, mask = self.random_transformer.random_transform(image, mask)
if self.stretch_and_mean:
mean_band = mean_bands[city]
for band in range(len(mean_band)):
image[..., band] -= mean_band[band]
if self.crop_shape is not None:
crops = 0
tries = 0
while crops < self.crops_per_image:
tries += 1
if self.random_transformer is None:
y_start = (ori_height - self.crop_shape[0]) // 2
x_start = (ori_width - self.crop_shape[1]) // 2
else:
y_start = random.randint(0, ori_height - self.crop_shape[0] - 1)
x_start = random.randint(0, ori_width - self.crop_shape[1] - 1)
y_end = y_start + self.crop_shape[0]
x_end = x_start + self.crop_shape[1]
crop_image = image[y_start:y_end, x_start:x_end, :]
crop_mask = mask[y_start:y_end, x_start:x_end, :]
if self.random_transformer is None:
batch_x.append(crop_image)
batch_y.append(crop_mask)
crops += 1
elif np.count_nonzero(crop_image) > 100 or tries > 20:
batch_x.append(crop_image)
batch_y.append(crop_mask)
crops += 1
else:
batch_x.append(image)
batch_y.append(mask)
batch_x = np.array(batch_x, dtype="float32")
batch_y = np.array(batch_y, dtype="float32")
if self.preprocessing_function == 'caffe':
batch_x_rgb = batch_x[..., :3]
batch_x_bgr = batch_x_rgb[..., ::-1]
batch_x[..., :3] = batch_x_bgr
if not self.stretch_and_mean:
batch_x = batch_x / 8. - 127.5
else:
if self.stretch_and_mean:
batch_x = batch_x / 255
else:
batch_x = batch_x / 1024. - 1
return self.transform_batch_x(batch_x), self.transform_batch_y(batch_y)
