def handle(self, **options):
shape_file = options['shape'][0]
sensor = options['sensor'][0]
country = options['country'][0]
column = options['column'][0]
country_object = Country.objects.get(name=country)
with fiona.open(shape_file) as source:
print(source.crs)
for feat in source:
s1 = shape(feat['geometry'])
if source.crs['init'] != 'epsg:4326':
project = partial(
pyproj.transform,
pyproj.Proj(source.crs),
pyproj.Proj(init='EPSG:4326'))
s2 = transform(project, s1)
else:
s2 = s1
geom = GEOSGeometry(s2.wkt)
if country_object.the_geom.intersects(geom):
name = '%03d%03d' % (feat['properties']['PATH'], feat['properties']['ROW'])
if name:
try:
o = Footprint(the_geom = geom, sensor=sensor, name=name)
o.save()
except IntegrityError:
logger.error('Name %s already exists.' % name)
def parse_args(self, args):
#program_name = os.path.basename(sys.argv[0])
parser = argparse.ArgumentParser() # description='', usage = ''
#parser.add_argument("-w", "--worker", default=None, help="worker (i/n)")
parser.add_argument("--name",
type=str,
default=None,
help="base name for output")
parser.add_argument("--center",
type=str,
default=None,
help="center of target area")
parser.add_argument("--area",
type=str,
default=None,
help="target area polygon GeoJSON")
#parser.add_argument("--radius", type=float, default=None, help="radius of target area")
#parser.add_argument("--area", type=str, help="GeoJSON polygon of target area")
#parser.add_argument("--tile", type=float, help="tile size in meters (0 for entire area)")
args = parser.parse_args(args)
center = args.center.split(",")
self.center = (float(center[0]), float(center[1]))
self.name = args.name
self.area = shape(json.loads(args.area))
def calculate_neighbors(state_abbr, year):
with open(f'{state_abbr.upper()}{sep}2017{sep}{state_abbr}_zoom7_0.geojson'
) as f:
precincts = json.load(f)
pshapes = {}
for pfeature in precincts['features']:
pshapes[pfeature['properties']['id']] = geo.shape(pfeature['geometry'])
all_neighbors = {}
# create a dictionary of precinct id keys and lists of ids of neighbors
# will be saved in a separate file not with GeoJSON since this data is
# not stored in the master DB it will have to be transmitted separately
# so keeping it separate will be helpful
for (outer_id, shape) in pshapes.items():
print(f"Finding neighbors for {outer_id}")
for (inner_id, candidate) in pshapes.items():
# note that touches alone is not enough
try:
if (shape.touches(candidate) or shape.intersects(candidate)
or shape.overlaps(candidate)):
if outer_id in all_neighbors:
all_neighbors[outer_id].append(inner_id)
else:
all_neighbors[outer_id] = [inner_id]
except shapely.errors.TopologicalError as e:
print(f'Error in finding neighbors for {outer_id}.')
print(e)
with open(
f'{state_abbr.upper()}{sep}{year}{sep}{state_abbr}_p_neighbors.scsv',
'w') as results_file:
for (pid, neighbor_list) in all_neighbors.items():
results_file.write(f'{pid}|{",".join(neighbor_list)}\n')
def calculate_border_precincts(state_abbr, year):
with open(f'{state_abbr.upper()}{sep}2017{sep}{state_abbr}_zoom7_0.geojson'
) as f:
precincts = json.load(f)
pshapes = {}
for pfeature in precincts['features']:
try:
precinct = Precinct(
pfeature['properties']['id'],
pfeature['properties']['containing_district_id'],
geo.shape(pfeature['geometry']))
pshapes[pfeature['properties']['id']] = precinct
except KeyError as e:
id = pfeature['properties'][
'id'] if 'properties' in pfeature and 'id' in pfeature[
'properties'] else '?'
print(f"Error on precinct: {id} -> {e}")
all_neighbors = {}
# dictionary relating district ids to lists of border precinct ids
border_precincts = set(map(lambda p: p.district_id, pshapes.values()))
border_precincts = {k: [] for k in border_precincts}
# all neighbors is a dictionary that maps precinct objects to lists
# of objects that are their neighbors
for (outer_id, precinct) in pshapes.items():
print(f"Finding neighbors for {outer_id}")
for (inner_id, candidate_precinct) in pshapes.items():
try:
# note that touches alone is not enough
if (precinct.shape.touches(candidate_precinct.shape)
or precinct.shape.intersects(candidate_precinct.shape)
or precinct.shape.overlaps(candidate_precinct.shape)):
if precinct in all_neighbors:
all_neighbors[precinct].append(candidate_precinct)
else:
all_neighbors[precinct] = [candidate_precinct]
except shapely.errors.TopologicalError as e:
print(f'Error in finding neighbors for {outer_id}.')
print(e)
# run through all the neighbors of each precinct and if any of them
# are in a district that is not the same as the precinct we call that
# a border precinct
for (precinct, neighbors) in all_neighbors.items():
if any(i.district_id != precinct.district_id for i in neighbors):
# if this is true then this precinct `precinct` is a border
# precinct within its district
border_precincts[precinct.district_id].append(precinct)
with open(
f'{state_abbr.upper()}{sep}{year}{sep}{state_abbr}_border_precincts.scsv',
'w') as f:
for (did, precincts) in border_precincts.items():
f.write(
f'{did}|{",".join(list(map(lambda p: p.id, precincts)))}\n')
def fromfile(cls, data, file_path):
kwargs = super().fromfile(data, file_path)
if 'geometry' not in data:
raise ValueError('missing geometry.')
try:
kwargs['geometry'] = shape(data['geometry'])
except:
raise ValueError(_('Invalid GeoJSON.'))
return kwargs
def storePolygons(jsonfile):
with open(jsonfile) as data_file:
js = json.load(data_file)
list_of_polygons = {}
for feature in js['features']:
if feature['geometry']:
polygon_id = feature['properties']['SA2_MAIN11']
list_of_polygons[polygon_id] = shape(feature['geometry'])
return list_of_polygons
def spans_feature(self, rect, ref_shape):
min_x, min_y, max_x, max_y = rect
tile_ulp = (min_x, max_y)
tile_dlp = (min_x, min_y)
tile_drp = (max_x, min_y)
tile_urp = (max_x, max_y)
tile = Polygon([tile_ulp, tile_dlp, tile_drp, tile_urp])
#return not tile.intersection(geom).is_empty
for feature in ref_shape:
shp_geom = shape(feature['geometry'])
poly_int =tile.intersection(shp_geom)
if not poly_int.is_empty:
return True
return False
def count_polygons(shp):
count=0
for feature in shp:
geom=shape(feature['geometry'])
print( feature['properties'])
if geom.type == 'Polygon':
count+=1
elif geom.type == 'MultiPolygon':
for part in geom:
count+=1
else:
raise ValueError('Unhandled geometry type: ' + repr(geom.type))
print( count)
return count
def _iter_source_(self):
with fiona.open(self.path, 'r') as source:
for feature in source:
# ensure the feature is valid
# https://github.com/Toblerity/Fiona/blob/master/examples/with-shapely.py
geom = shape(feature['geometry'])
try:
if not geom.is_valid:
clean = geom.buffer(0.0)
geom = clean
feature['geometry'] = mapping(geom)
assert clean.is_valid
assert (clean.geom_type == 'Polygon')
except (AssertionError, AttributeError) as e:
warn('{2}. Invalid geometry found with id={0} and properties: {1}'.format(feature['id'],
feature['properties'],
e))
feature['shapely'] = geom
yield feature
def _iter_source_(self):
with fiona.open(self.path, 'r') as source:
for feature in source:
# ensure the feature is valid
# https://github.com/Toblerity/Fiona/blob/master/examples/with-shapely.py
geom = shape(feature['geometry'])
try:
if not geom.is_valid:
clean = geom.buffer(0.0)
geom = clean
feature['geometry'] = mapping(geom)
assert clean.is_valid
assert (clean.geom_type == 'Polygon')
except (AssertionError, AttributeError) as e:
warn('{2}. Invalid geometry found with id={0} and properties: {1}'.format(feature['id'],
feature['properties'],
e))
feature['shapely'] = geom
yield feature
def handle(self, **options):
path = options['path'][0]
column = options['column'][0]
model = options['model'][0]
with fiona.open(path) as src:
print json.dumps(src.schema, indent=4)
print src.crs
for feat in src:
#print feat['geometry']['type']
s = shape(feat['geometry'])
if feat['geometry']['type'] == 'Polygon':
s = MultiPolygon([s])
print json.dumps(feat['geometry'])
klass = locate('madmex.models.%s' % model)
f = klass(name=feat['properties'][column],
the_geom=GEOSGeometry(s.wkt))
f.save()
def get_2d_state_boundaries(self):
geoms = []
build = True
with fiona.open('/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/shp/state_boundaries/state_boundaries.shp','r') as source:
for ii,row in enumerate(source):
if build:
nrows = len(source)
dtype = []
for k,v in source.schema['properties'].iteritems():
if v.startswith('str'):
v = str('|S{0}'.format(v.split(':')[1]))
else:
v = getattr(np,v.split(':')[0])
dtype.append((str(k),v))
fill = np.empty(nrows,dtype=dtype)
ref_names = fill.dtype.names
build = False
fill[ii] = tuple([row['properties'][n] for n in ref_names])
geoms.append(shape(row['geometry']))
geoms = np.atleast_2d(geoms)
return(geoms,fill)
def _iter_source_(self):
with fiona.open(self.path, "r") as source:
for feature in source:
## ensure the feature is valid
## https://github.com/Toblerity/Fiona/blob/master/examples/with-shapely.py
try:
geom = shape(feature["geometry"])
if not geom.is_valid:
clean = geom.buffer(0.0)
geom = clean
feature["geometry"] = mapping(geom)
assert clean.is_valid
assert clean.geom_type == "Polygon"
except (AssertionError, AttributeError) as e:
warn(
"{2}. Invalid geometry found with id={0} and properties: {1}".format(
feature["id"], feature["properties"], e
)
)
feature["shapely"] = geom
yield (feature)
def sensors(db, provider, request):
import json
import os.path
from shapely.geometry.geo import shape
from app.models import Collection, Item
collection = Collection.create(name="sensors",
is_public=True,
provider_uuid=provider["uuid"])
my_path = os.path.abspath(os.path.dirname(__file__))
path = os.path.join(my_path, "../../data/sensors.json")
with open(path) as f:
data = json.load(f)
for feature in data["features"]:
Item.create(
collection_uuid=collection.uuid,
properties=feature["properties"],
geometry=shape(feature["geometry"]).to_wkt(),
)
Collection.session.commit()
return collection.to_dict()
def get_2d_state_boundaries(self):
geoms = []
build = True
sc = GeomCabinet()
path = sc.get_shp_path('state_boundaries')
with fiona.open(path, 'r') as source:
for ii, row in enumerate(source):
if build:
nrows = len(source)
dtype = []
for k, v in source.schema['properties'].iteritems():
if v.startswith('str'):
v = str('|S{0}'.format(v.split(':')[1]))
else:
v = getattr(np, v.split(':')[0])
dtype.append((str(k), v))
fill = np.empty(nrows, dtype=dtype)
ref_names = fill.dtype.names
build = False
fill[ii] = tuple([row['properties'][n] for n in ref_names])
geoms.append(shape(row['geometry']))
geoms = np.atleast_2d(geoms)
return geoms, fill
def get_2d_state_boundaries(self):
geoms = []
build = True
sc = GeomCabinet()
path = sc.get_shp_path('state_boundaries')
with fiona.open(path, 'r') as source:
for ii, row in enumerate(source):
if build:
nrows = len(source)
dtype = []
for k, v in source.schema['properties'].items():
if v.startswith('str'):
v = str('|S{0}'.format(v.split(':')[1]))
else:
v = getattr(np, v.split(':')[0])
dtype.append((str(k), v))
fill = np.empty(nrows, dtype=dtype)
ref_names = fill.dtype.names
build = False
fill[ii] = tuple([row['properties'][n] for n in ref_names])
geoms.append(shape(row['geometry']))
geoms = np.atleast_2d(geoms)
return geoms, fill
def osm_init_params(root, pipeline, logger):
"""
Process OSM build parameters.
"""
if pipeline.data.get('ddd:osm:area', None):
# Use provided shape
area_shape = shape(json.loads(pipeline.data.get('ddd:osm:area')))
pipeline.data['ddd:osm:area:shape'] = area_shape
pipeline.data['ddd:osm:area:radius'] = None
if pipeline.data.get('ddd:osm:area:radius', None):
# Convert radius to meters
pipeline.data['ddd:osm:area:radius'] = parse_meters(
pipeline.data['ddd:osm:area:radius'])
pipeline.data['ddd:osm:area:shape'] = None
if pipeline.data.get('ddd:osm:area:chunk_size', None):
pipeline.data['ddd:osm:area:chunk_size'] = parse_meters(
pipeline.data['ddd:osm:area:chunk_size'])
xyztile = pipeline.data.get('ddd:osm:area:xyztile', None)
if xyztile:
if ('ddd:osm:area:radius' in pipeline.data
or 'ddd:osm:area:center' in pipeline.data
or 'ddd:osm:area:size' in pipeline.data):
logger.error(
"Option --xyztile cannot be used with --radius, --center or --size ."
)
sys.exit(2)
pipeline.data['ddd:osm:area:xyztile'] = parse_tile(xyztile)
if pipeline.data.get('ddd:osm:area:center', None):
center = pipeline.data['ddd:osm:area:center'].split(",")
center = (float(center[0]), float(center[1]))
pipeline.data['ddd:osm:area:center'] = center
def countries_intersecting(boundary: Geometry):
bounds = shape(boundary)
subset = gdf.loc[gdf.intersects(bounds)]
return rows_to_countries(subset)
async def process_source(filename: str, session: ClientSession):
try:
async with aiofiles.open(filename, mode="r", encoding="utf-8") as f:
contents = await f.read()
source = json.loads(contents)
# Exclude sources
# Skip non wms layers
if not source["properties"]["type"] == "wms":
return
# check if it is esri rest and not wms
if "bboxSR" in source["properties"]["url"]:
return
if "available_projections" not in source["properties"]:
return
if "header" in source["properties"]["url"]:
return
if "geometry" not in source:
return
processed_sources.add(filename)
category = source["properties"].get("category", None)
if source["geometry"] is None:
geom: MultiPolygon | Polygon = box(-180, -90, 180, 90)
pt: Point = Point(7.44, 46.56)
else:
geom = shape(source["geometry"])
pt: Point = geom.representative_point() # type: ignore
test_zoom_level = ZOOM_LEVEL
if "min_zoom" in source["properties"]:
test_zoom_level = max(source["properties"]["min_zoom"],
test_zoom_level)
if "max_zoom" in source["properties"]:
test_zoom_level = min(source["properties"]["max_zoom"],
test_zoom_level)
old_url = source["properties"]["url"]
old_projections = source["properties"]["available_projections"]
# Get existing image hash
original_img_messages: List[str] = []
original_image_result = await get_image(
url=old_url,
available_projections=old_projections,
lon=pt.x, # type: ignore
lat=pt.y, # type: ignore
zoom=test_zoom_level,
session=session,
messages=original_img_messages,
)
if not original_image_result.status == ImageHashStatus.SUCCESS or original_image_result.image_hash is None: # type: ignore
ignored_sources[
filename] = "Not possible to download reference image"
# We are finished if it was not possible to get the image
return
if max_count(str(
original_image_result.image_hash)) == 16: # type: ignore
if "category" in source["properties"] and "photo" in source[
"properties"]["category"]:
msgs = "\n\t".join(original_img_messages)
logging.warning(
f"{filename}: has category {category} but image hash is {original_image_result.image_hash}:\n\t{msgs}"
) # type: ignore
# These image hashes indicate that the downloaded image is not useful to determine
# if the updated query returns the same image
error_msgs = "\n\t".join(original_img_messages)
logging.warning(
f"{filename}: Image hash {original_image_result.image_hash} not useful ({category}): \n\t{error_msgs}"
) # type: ignore
ignored_sources[
filename] = f"Image hash {original_image_result.image_hash} not useful ({category})" # type: ignore
return
# Update wms
wms_messages: List[str] = []
result = await update_wms(old_url, session, wms_messages)
if result is None:
error_msgs = "\n\t".join(wms_messages)
logging.info(
f"{filename}: Not possible to update wms url:\n\t{error_msgs}")
ignored_sources[filename] = "Not possible to update wms url"
return
new_url, new_projections = result
del result
# Servers that report a lot of projection may be configured wrongly
# Check for CRS:84, EPSG:3857, EPSG:4326 and keep existing projections if still advertised
if len(new_projections) > 15:
filtered_projs: Set[str] = set()
for proj in ["CRS:84", "EPSG:3857", "EPSG:4326"]:
if proj in new_projections:
filtered_projs.add(proj)
for proj in old_projections:
if proj in new_projections:
filtered_projs.add(proj)
new_projections = filtered_projs
# Download image for updated url
new_img_messages: List[str] = []
updated_image_result = await get_image(
url=new_url,
available_projections=new_projections,
lon=pt.x, # type: ignore
lat=pt.y, # type: ignore
zoom=test_zoom_level,
session=session,
messages=new_img_messages,
)
if not updated_image_result.status == ImageHashStatus.SUCCESS or updated_image_result.image_hash is None: # type: ignore
error_msgs = "\n\t".join(new_img_messages)
logging.warning(
f"{filename}: Could not download image with updated url: {updated_image_result.status}\n\t{error_msgs}"
)
ignored_sources[
filename] = "Could not download image with updated url"
return
# Only sources are updated where the new query returns the same image
if not image_similar(original_image_result.image_hash,
updated_image_result.image_hash,
test_zoom_level): # type: ignore
original_hash = original_image_result.image_hash # type: ignore
new_hash = updated_image_result.image_hash # type: ignore
hash_diff = original_hash - new_hash # type: ignore
error_original_img_messages = "\n\t".join(original_img_messages)
error_new_img_messages = "\n\t".join(new_img_messages)
logging.info(
f"{filename}: ImageHash not the same for: {filename}: {original_hash} - {new_hash}: {hash_diff}\n\t{error_original_img_messages} \n\t{error_new_img_messages}"
)
ignored_sources[
filename] = f"ImageHash for reference image and image with updated url differs: {original_hash} - {new_hash}: {new_hash}"
return
# Test if selected projections work despite not being advertised
for EPSG in {"EPSG:3857", "EPSG:4326"}:
if EPSG not in new_projections:
epsg_check_messages: List[str] = []
epsg_image_result = await get_image(
url=new_url,
available_projections=[EPSG],
lon=pt.x, # type: ignore
lat=pt.y, # type: ignore
zoom=test_zoom_level,
session=session,
messages=epsg_check_messages,
)
epsg_check_messages_str = "\n\t".join(epsg_check_messages)
logging.info(
f"{filename}: Test if projection {EPSG} works despite not advertised:\n\t{epsg_check_messages_str}"
)
if epsg_image_result.status == ImageHashStatus.NETWORK_ERROR:
if EPSG in old_projections and EPSG not in new_projections:
new_projections.add(EPSG)
added_projections[filename][
"Network error, but projection was previously included."].append(
EPSG)
elif epsg_image_result.status == ImageHashStatus.SUCCESS:
epsg_image_hash = epsg_image_result.image_hash # type: ignore
original_image_hash = original_image_result.image_hash # type: ignore
# Relax similarity constraint to account for differences due to loss of quality due to re-projection
hash_diff = original_image_result.image_hash - epsg_image_result.image_hash # type: ignore
if image_similar(original_image_result.image_hash,
epsg_image_result.image_hash,
test_zoom_level): # type: ignore
new_projections.add(EPSG)
added_projections[filename][
"Projection returns similar image despite not advertised."].append(
EPSG)
logging.info(
f"{filename}: Add {EPSG} despite not being advertised: {epsg_image_hash} - {original_image_hash}: {hash_diff}"
)
elif epsg_image_hash is not None:
logging.info(
f"{filename}: Do not add {EPSG} Difference: {epsg_image_hash} - {original_image_hash}: {hash_diff}"
)
else:
logging.info(
f"{filename}: Do not add {EPSG} No image returned."
)
# Check if projections are supported by server
not_supported_projections: Set[str] = set()
image_hashes: Dict[str, Tuple[ImageResult, List[str]]] = {}
for proj in new_projections:
proj_messages: List[str] = []
epsg_image_result = await get_image(
url=new_url,
available_projections=[proj],
lon=pt.x, # type: ignore
lat=pt.y, # type: ignore
zoom=test_zoom_level,
session=session,
messages=proj_messages,
)
image_hashes[proj] = (epsg_image_result, proj_messages)
msgs = "\n\t".join(proj_messages)
logging.info(
f"{filename} Projection check: {proj}: {epsg_image_result.status}:\n\t{msgs}"
)
if epsg_image_result.status == ImageHashStatus.IMAGE_ERROR:
not_supported_projections.add(proj)
removed_projections[filename][
"Projection check: does not return an image"].append(proj)
elif epsg_image_result.status == ImageHashStatus.NETWORK_ERROR:
# If not successfully status do not add if not previously added
if proj not in old_projections:
removed_projections[filename][
"Projection check: network error and previously not included"].append(
proj)
not_supported_projections.add(proj)
if len(not_supported_projections) > 0:
removed = ",".join(not_supported_projections)
logging.info(
f"{filename}: remove projections that are advertised but do not return an image: {removed}"
)
new_projections -= not_supported_projections
# Check if EPSG:3857 and EPSG:4326 are similar
if ("EPSG:3857" in image_hashes and "EPSG:4326" in image_hashes and
image_hashes["EPSG:3857"][0].status == ImageHashStatus.SUCCESS
and image_hashes["EPSG:4326"][0].status
== ImageHashStatus.SUCCESS):
img_hash_3857 = image_hashes["EPSG:3857"][
0].image_hash # type: ignore
img_hash_4326 = image_hashes["EPSG:4326"][
0].image_hash # type: ignore
diff_hash = img_hash_3857 - img_hash_4326 # type: ignore
if not image_similar(img_hash_3857, img_hash_4326,
test_zoom_level):
msgs = "\n\t".join(image_hashes["EPSG:3857"][1] +
image_hashes["EPSG:4326"][1])
logging.warning(
f"{filename}: ({category}) ImageHash for EPSG:3857 and EPSG:4326 not similar: {img_hash_3857} - {img_hash_4326}: {diff_hash}:\n\t{msgs}"
)
# Check projections again to filter out EPSG:3857 alias
new_projections = eliutils.clean_projections(new_projections)
# Check if only formatting has changed
url_has_changed = not compare_urls(source["properties"]["url"],
new_url)
projections_have_changed = not compare_projs(
source["properties"]["available_projections"],
new_projections,
)
if url_has_changed:
source["properties"]["url"] = new_url
if projections_have_changed:
source["properties"]["available_projections"] = list(
sorted(
new_projections,
key=lambda x: (x.split(":")[0], int(x.split(":")[1])),
))
if url_has_changed or projections_have_changed:
with open(filename, "w", encoding="utf-8") as out:
json.dump(source,
out,
indent=4,
sort_keys=False,
ensure_ascii=False)
out.write("\n")
except Exception as e:
logging.exception(
f"{filename}: Error occurred while processing source: {e}")
_x, _y = pyproj.transform(proj_lv03, proj_lv95, x, y)
bbox = (_x - D, _y - D, _x + D, _y + D)
crs = CRS.from_epsg(2056)
else:
src = fiona.open('/home/rbuffat/buildings_av_osm_tlm.shp')
bbox = (x - D, y - D, x + D, y + D)
crs = CRS.from_epsg(21781)
proj = pyproj.Proj("+init=EPSG:21781")
geom = None
for ss in src.filter(bbox=bbox):
if ss['properties']['bid'] == int(
row['bid']
) and ss['properties']['btype'] == row['btype']:
geom = shape(ss['geometry'])
# get building area
barea = geom.area
hor_sis = []
# get hor solar irad
lon, lat = pyproj.transform(proj_lv03,
pyproj.Proj("+init=EPSG:4326"), x, y)
d = datetime.date(2017, 1, 1)
while d.year < 2018:
# logging.info("{}_{}: {}".format(btype, bid, str(d)))
year = d.year
month = d.month
day = d.day
def check_tms(source: Dict[str, Any], messages: List[Message]) -> None:
"""Check TMS source
Parameters
----------
source : Dict[str, Any]
The source
messages : List[Message]
The list to add messages to
"""
try:
url = source["properties"]["url"]
source_headers = get_http_headers(source)
if source["geometry"] is None:
geom = None
else:
geom = shape(source["geometry"])
# Validate URL
try:
_url = re.sub(r"switch:?([^}]*)", "switch",
url).replace("{", "").replace("}", "")
validators.url(_url) # type: ignore
except validators.utils.ValidationFailure as e:
messages.append(
Message(level=MessageLevel.ERROR,
message=f"URL validation error {e} / {url}"))
# Check URL parameter
parameters = {}
# {z} instead of {zoom}
if "{z}" in source["properties"]["url"]:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Parameter {{z}} is used instead of {{zoom}} in tile url: {url}"
))
return
# We can't test sources that have an apikey, that is unknown to ELI
if "{apikey}" in url:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Not possible to check URL, apikey is required: {url}"))
return
# If URL contains a {switch:a,b,c} parameters, use the first for tests
match = re.search(r"switch:?([^}]*)", url)
if match is not None:
switches = match.group(1).split(",")
url = url.replace(match.group(0), "switch")
parameters["switch"] = switches[0]
# Check zoom levels
min_zoom = 0
max_zoom = 22
if "min_zoom" in source["properties"]:
min_zoom = int(source["properties"]["min_zoom"])
if "max_zoom" in source["properties"]:
max_zoom = int(source["properties"]["max_zoom"])
# Check if we find a TileMap Resource to check for zoom levels
# While there is a typical location for metadata, there is no requirement
# that the metadata need to be located there.
tms_url = tmshelper.TMSURL(url=url)
tilemap_resource_url = tms_url.get_tilemap_resource_url()
if tilemap_resource_url is not None:
for tilemap_url in [
tilemap_resource_url,
tilemap_resource_url + "/tilemapresource.xml",
]:
try:
r, xml = get_text_encoded(tilemap_url.format(**parameters),
headers=headers)
if r.status_code == 200 and xml is not None:
try:
tilemap_resource = tmshelper.TileMapResource(xml)
except Exception:
# Not all TMS server provide TileMap resources.
continue
if tilemap_resource.tile_map is None:
continue
# Check zoom levels against TileMapResource
tilemap_minzoom, tilemap_maxzoom = tilemap_resource.get_min_max_zoom_level(
)
if min_zoom == tilemap_minzoom:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"min_zoom level '{min_zoom}' not the same as specified in TileMap: '{tilemap_minzoom}': {tilemap_url}. "
"Caution: this might be intentional as some server timeout for low zoom levels.",
))
if not max_zoom == tilemap_maxzoom:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"max_zoom level '{max_zoom}' not the same as specified in TileMap: '{tilemap_maxzoom}': {tilemap_url}",
))
# Check geometry within bbox
if geom is not None and tilemap_resource.tile_map.bbox84 is not None:
max_area_outside = max_area_outside_bbox(
geom, tilemap_resource.tile_map.bbox84)
# 5% is an arbitrary chosen value and should be adapted as needed
if max_area_outside > 5.0:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"{round(max_area_outside, 2)}% of geometry is outside of the layers bounding box. Geometry should be checked",
))
break
except Exception as e:
print(f"Error fetching TMS: {e}: {url}")
pass
# Test zoom levels by accessing tiles for a point within the geometry
if geom is not None:
centroid: Point = geom.representative_point() # type: ignore
else:
centroid = Point(6.1, 49.6)
centroid_x: float = centroid.x # type: ignore
centroid_y: float = centroid.y # type: ignore
zoom_failures: List[Tuple[int, str, int, Optional[str]]] = []
zoom_success: List[int] = []
tested_zooms: Set[int] = set()
def test_zoom(zoom: int) -> None:
tested_zooms.add(zoom)
tile: mercantile.Tile = mercantile.tile(centroid_x, centroid_y,
zoom) # type: ignore
tile_x: int = tile.x # type: ignore
tile_y: int = tile.y # type: ignore
query_url = url
if "{-y}" in url:
y = 2**zoom - 1 - tile_y
query_url = query_url.replace("{-y}", str(y))
elif "{!y}" in url:
y = 2**(zoom - 1) - 1 - tile_y
query_url = query_url.replace("{!y}", str(y))
else:
query_url = query_url.replace("{y}", str(tile_y))
parameters["x"] = tile_x
parameters["zoom"] = zoom
query_url = query_url.format(**parameters)
url_is_good, http_code, mime = test_image(query_url,
source_headers)
if url_is_good:
zoom_success.append(zoom)
else:
zoom_failures.append((zoom, query_url, http_code, mime))
# Test zoom levels
for zoom in range(min_zoom, max_zoom + 1):
test_zoom(zoom)
tested_str = ",".join(list(map(str, sorted(tested_zooms))))
sorted_failures = sorted(zoom_failures, key=lambda x: x[0])
if len(zoom_failures) == 0 and len(zoom_success) > 0:
messages.append(
Message(
level=MessageLevel.INFO,
message=f"Zoom levels reachable. (Tested: {tested_str})"))
elif len(zoom_failures) > 0 and len(zoom_success) > 0:
not_found_str = ",".join(
list(map(str, [level for level, _, _, _ in sorted_failures])))
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Zoom level {not_found_str} not reachable. (Tested: {tested_str}) Tiles might not be present at tested location: {centroid_x},{centroid_y}",
))
for level, url, http_code, mime_type in sorted_failures:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"URL for zoom level {level} returned HTTP Code {http_code}: {url} MIME type: {mime_type}",
))
else:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"No zoom level reachable. (Tested: {tested_str}) Tiles might not be present at tested location: {centroid_x},{centroid_y}",
))
for level, url, http_code, mime_type in sorted_failures:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"URL for zoom level {level} returned HTTP Code {http_code}: {url} MIME type: {mime_type}",
))
except Exception as e:
messages.append(
Message(
level=MessageLevel.ERROR,
message=f"Failed testing TMS source: Exception: {e}",
))
def check_wms(source: Dict[str, Any], messages: List[Message]) -> None:
"""Check WMS source
Parameters
----------
source : Dict[str, Any]
The source
messages : List[Message]
The list to add messages to
"""
url = source["properties"]["url"]
wms_url = wmshelper.WMSURL(url)
source_headers = get_http_headers(source)
params = ["{proj}", "{bbox}", "{width}", "{height}"]
missingparams = [p for p in params if p not in url]
if len(missingparams) > 0:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"The following values are missing in the URL: {','.join(missingparams)}",
))
try:
wms_url.is_valid_getmap_url()
except validators.utils.ValidationFailure as e:
messages.append(
Message(level=MessageLevel.ERROR,
message=f"URL validation error {e} for {url}"))
# Check mandatory WMS GetMap parameters (Table 8, Section 7.3.2, WMS 1.3.0 specification)
# Normalize parameter names to lower case
wms_args = {key.lower(): value for key, value in wms_url.get_parameters()}
# Check if it is actually a ESRI Rest url and not a WMS url
is_esri = "request" not in wms_args
# Check if required parameters are missing
missing_request_parameters: Set[str] = set()
if is_esri:
required_parameters = [
"f", "bbox", "size", "imageSR", "bboxSR", "format"
]
else:
required_parameters = [
"version",
"request",
"layers",
"bbox",
"width",
"height",
"format",
]
for request_parameter in required_parameters:
if request_parameter.lower() not in wms_args:
missing_request_parameters.add(request_parameter)
if not is_esri:
if "version" in wms_args and wms_args["version"] == "1.3.0":
if "crs" not in wms_args:
missing_request_parameters.add("crs")
if "srs" in wms_args:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"WMS {wms_args['version']} URLs should not contain SRS parameter: {url}",
))
elif "version" in wms_args and not wms_args["version"] == "1.3.0":
if "srs" not in wms_args:
missing_request_parameters.add("srs")
if "crs" in wms_args:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"WMS {wms_args['version']} URLs should not contain CRS parameter: {url}",
))
if len(missing_request_parameters) > 0:
missing_request_parameters_str = ",".join(missing_request_parameters)
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Parameter '{missing_request_parameters_str}' is missing in URL: {url}.",
))
return
# Nothing more to do for ESRI Rest API
if is_esri:
return
# Styles is mandatory according to the WMS specification, but some WMS servers seems not to care
if "styles" not in wms_args:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Parameter 'styles' is missing in url. 'STYLES=' can be used to request default style.: {url}",
))
# We first send a service=WMS&request=GetCapabilities request to server
# According to the WMS Specification Section 6.2 Version numbering and negotiation, the server should return
# the GetCapabilities XML with the highest version the server supports.
# If this fails, it is tried to explicitly specify a WMS version
exceptions: List[str] = []
wms = None
for wms_version in [None, "1.3.0", "1.1.1", "1.1.0", "1.0.0"]:
if wms_version is None:
wms_version_str = "-"
else:
wms_version_str = wms_version
wms_getcapabilities_url = None
try:
wms_getcapabilities_url = wms_url.get_capabilities_url(
wms_version=wms_version)
_, xml = get_text_encoded(wms_getcapabilities_url,
headers=source_headers)
if xml is not None:
wms = wmshelper.WMSCapabilities(xml)
break
except Exception as e:
exceptions.append(
f"WMS {wms_version_str}: Error: {e} {wms_getcapabilities_url}")
continue
# Check if it was possible to parse the WMS GetCapability response
# If not, there is nothing left to check
if wms is None:
for msg in exceptions:
messages.append(Message(
level=MessageLevel.ERROR,
message=msg,
))
return
# Log access constraints and fees metadata
for access_constraint in wms.access_constraints:
messages.append(
Message(
level=MessageLevel.INFO,
message=f"AccessConstraints: {access_constraint}",
))
for fee in wms.fees:
messages.append(
Message(
level=MessageLevel.INFO,
message=f"Fee: {fee}",
))
if source["geometry"] is None:
geom = None
else:
geom = shape(source["geometry"]) # type: ignore
# Check layers
if "layers" in wms_args:
layers = wms_args["layers"].split(",")
# Check if layers in WMS GetMap URL are advertised by WMS server.
not_found_layers = [
layer_name for layer_name in layers if layer_name not in wms.layers
]
if len(not_found_layers) > 0:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Layers '{','.join(not_found_layers)}' not advertised by WMS GetCapabilities request (Some server do not advertise layers, but they are very rare).: {url}",
))
# Check source geometry against layer bounding box
# Regardless of its projection, each layer should advertise an approximated bounding box in lon/lat.
# See WMS 1.3.0 Specification Section 7.2.4.6.6 EX_GeographicBoundingBox
if geom is not None and geom.is_valid: # type: ignore
bboxs = [
wms.layers[layer_name].bbox for layer_name in layers
if layer_name in wms.layers and wms.layers[layer_name].bbox
]
bboxs = [bbox for bbox in bboxs if bbox is not None]
max_area_outside = max_area_outside_bbox(geom,
bboxs) # type: ignore
# 5% is an arbitrary chosen value and should be adapted as needed
if max_area_outside > 5.0:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"{round(max_area_outside, 2)}% of geometry is outside of the layers bounding box. Geometry should be checked",
))
# Check styles
if "styles" in wms_args:
style_parameter = wms_args["styles"]
# default style needs not to be advertised by the server
if not (style_parameter == "default" or style_parameter == ""
or style_parameter == "," * len(layers)):
styles = style_parameter.split(",")
if not len(styles) == len(layers):
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Not the same number of styles and layers. {len(styles)} vs {len(layers)}",
))
else:
for layer_name, style_name in zip(layers, styles):
if (len(style_name) > 0 and not style_name == "default"
and layer_name in wms.layers and style_name
not in wms.layers[layer_name].styles):
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Layer '{layer_name}' does not support style '{style_name}'",
))
# Check CRS
if "available_projections" not in source["properties"]:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Sources of type wms must include the 'available_projections' element.",
))
else:
# A WMS server can include many CRS. Some of them are frequently used by editors. We require them to be included if they are supported by the WMS server.
crs_should_included_if_available = {
"EPSG:4326", "EPSG:3857", "CRS:84"
}
for layer_name in layers:
if layer_name in wms.layers:
# Check for CRS in available_projections that are not advertised by the WMS server
not_supported_crs: Set[str] = set()
available_projections: List[str] = source["properties"][
"available_projections"]
for crs in available_projections:
if crs.upper() not in wms.layers[layer_name].crs:
not_supported_crs.add(crs)
if len(not_supported_crs) > 0:
supported_crs_str = ",".join(
wms.layers[layer_name].crs)
not_supported_crs_str = ",".join(not_supported_crs)
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Layer '{layer_name}': CRS '{not_supported_crs_str}' not in: {supported_crs_str}. Some server support CRS which are not advertised.",
))
# Check for CRS supported by the WMS server but not in available_projections
supported_but_not_included: Set[str] = set()
for crs in crs_should_included_if_available:
if crs not in available_projections and crs in wms.layers[
layer_name].crs:
supported_but_not_included.add(crs)
if len(supported_but_not_included) > 0:
supported_but_not_included_str = ",".join(
supported_but_not_included)
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Layer '{layer_name}': CRS '{supported_but_not_included_str}' not included in available_projections but supported by server.",
))
# Check if server supports a newer WMS version as in url
if wms_args["version"] < wms.version:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Query requests WMS version '{wms_args['version']}', server supports '{wms.version}'",
))
# Check image formats
request_imagery_format = wms_args["format"]
wms_advertised_formats_str = "', '".join(wms.formats)
if request_imagery_format not in wms.formats:
messages.append(
Message(
level=MessageLevel.ERROR,
message=
f"Format '{request_imagery_format}' not in '{wms_advertised_formats_str}': {url}.",
))
# For photo sources it is recommended to use jpeg format, if it is available
if "category" in source["properties"] and "photo" in source["properties"][
"category"]:
if "jpeg" not in request_imagery_format and "jpeg" in wms.formats:
messages.append(
Message(
level=MessageLevel.WARNING,
message=
f"Server supports JPEG, but '{request_imagery_format}' is used. "
f"JPEG is typically preferred for photo sources, but might not be always "
f"the best choice. (Server supports: '{wms_advertised_formats_str}')",
))
# jsonschema validate
try:
validator.validate(source, schema)
except Exception as e:
messages.append(
Message(
level=MessageLevel.ERROR,
message=f"{filename} JSON validation error: {e}",
))
logger.info(f"Type: {source['properties']['type']}")
# Check geometry
if "geometry" in source:
geom = shape(source["geometry"]) # type: ignore
# Check if geometry is a valid (e.g. no intersection etc.)
if not geom.is_valid: # type: ignore
try:
reason = explain_validity(geom) # type: ignore
messages.append(
Message(
level=MessageLevel.ERROR,
message=f"{filename} invalid geometry: {reason}",
))
valid_geom = make_valid(geom) # type: ignore
valid_geom = eliutils.orient_geometry_rfc7946(
valid_geom) # type: ignore
valid_geom_json = json.dumps(
mapping(valid_geom),
#Transfer Variables
pow2_bounds = None
with fiona.open(args.input_shp, 'r', 'ESRI Shapefile') as input_shp_fh:
print(f"CRS: {input_shp_fh.crs}")
print(f"Features #: {str(len(input_shp_fh))}")
count=0
for data_feature in input_shp_fh:
if count > FEATURE_COUNT_LIMIT:
break
pprint(data_feature['properties'])
#get feature geometry
df_geom = shape(data_feature['geometry'])
print(f"Shapefile Geom bounds: {df_geom.bounds}")
pow2_bounds = get_pow2_extents(get_bounds_1x1km(df_geom.bounds), TILE_SIZE)
print(f"Bounds^2: {str(pow2_bounds)}")
"""
Gen QuadTree
"""
rootrect = list(pow2_bounds)
rootnode = Node(None, rootrect)
tree = QuadTree(rootnode, TILE_SIZE, df_geom)
print(f"Leaves: {str(len(tree.leaves))}")
out_shp_schema = {
def areNeighbors(geom1, geom2):
geom1 = geo.shape(geom1)
geom2 = geo.shape(geom2)
if geom1.intersects(geom2):
return True
return False
def parse_args(self, args):
#program_name = os.path.basename(sys.argv[0])
parser = argparse.ArgumentParser() # description='', usage = ''
#parser.add_argument("-w", "--worker", default=None, help="worker (i/n)")
parser.add_argument("-l",
"--limit",
type=int,
default=None,
help="tasks limit")
parser.add_argument("--name",
type=str,
default=None,
help="base name for output")
parser.add_argument("--center",
type=str,
default=None,
help="center of target area (lon, lat)")
parser.add_argument("--area",
type=str,
default=None,
help="target area polygon GeoJSON")
parser.add_argument("--radius",
type=float,
default=None,
help="radius of target area (m)")
parser.add_argument("--size",
type=float,
default=None,
help="tile size or 0 (m)")
parser.add_argument("--xyztile",
type=str,
default=None,
help="XYZ grid tile")
args = parser.parse_args(args)
self.limit = args.limit
self.name = args.name
if args.area:
self.area = shape(json.loads(args.area))
self._radius = None
else:
self.area = None
self._radius = args.radius
self.chunk_size = args.size
self.xyztile = None
if (args.xyztile):
if (args.radius or args.center or args.size):
logger.error(
"Option --xyztile cannot be used with --radius, --center or --size ."
)
sys.exit(2)
x, y, z = args.xyztile.split(",")
self.xyztile = int(x), int(y), int(z)
else:
center = args.center.split(",")
self.center = (float(center[0]), float(center[1]))
def load_geojson(self, files):
features = []
for f in files:
fs = geojson.load(open(f, 'r'))
features.extend(fs['features'])
seen = set()
dedup = []
features_custom = []
for f in features:
#oid = hash(str(f)) # f['properties']['osm_id']
# TODO: better way to distinguish custom features
if 'id' not in f:
features_custom.append(f)
continue
oid = f['id']
if oid not in seen:
seen.add(oid)
dedup.append(f)
logger.info("Loaded %d features (%d unique)" %
(len(features), len(dedup)))
features = dedup
# Project to local
# TODO: Do this with self.project coordinates after creating shapes (TEST!)
transformer = pyproj.Transformer.from_proj(self.osm_proj,
self.ddd_proj)
for f in features:
if not f['geometry']['coordinates']:
logger.info("Feature with no coordinates: %s", f)
else:
try:
f['geometry']['coordinates'] = project_coordinates(
f['geometry']['coordinates'], transformer)
except Exception as e:
logger.error("Cannot project coordinates for: %s", f)
# Filter "supertile"
filtered = []
for f in features:
if not f['geometry']['coordinates']: continue
try:
geom = shape(f['geometry'])
except Exception as e:
logger.warn(
"Could not load feature with invalid geometry (%s): %s",
str(f.properties)[:240], e)
continue
#geom = make_valid(geom)
#if self.area_filter.contains(geom.centroid):
try:
if self.area_filter.intersects(geom):
filtered.append(f)
except Exception as e:
logger.debug(
"Could not load feature (1/2) with invalid geometry (%s): %s",
str(f.properties)[:240], e)
# Attempt intersection first
try:
geom = geom.intersection(self.area_filter)
if self.area_filter.intersects(geom):
filtered.append(f)
except Exception as e:
logger.warn(
"Could not load feature (2/2) with invalid geometry (%s): %s",
f.properties, e)
continue
features = filtered
logger.info("Using %d features after filtering to %s" %
(len(features), self.area_filter.bounds))
self.features = features
self.features_custom = features_custom
def plot_flow_directions_and_basin_boundaries(ax, s, sim_name_to_station_to_model_point,
sim_name_to_manager=None):
assert isinstance(ax, Axes)
assert isinstance(s, Station)
assert isinstance(sim_name_to_station_to_model_point, dict)
mp_list = list(sim_name_to_station_to_model_point.items())[0][1][s]
#selecting only one (the first model point)
mp = mp_list[0]
manager = list(sim_name_to_manager.items())[0][1]
assert isinstance(manager, Crcm5ModelDataManager)
flow_in_mask = manager.get_mask_for_cells_upstream(mp.ix, mp.jy)
lons2d, lats2d = manager.lons2D, manager.lats2D
i_upstream, j_upstream = np.where(flow_in_mask == 1)
nx_total, ny_total = lons2d.shape
imin, imax = np.min(i_upstream), np.max(i_upstream)
jmin, jmax = np.min(j_upstream), np.max(j_upstream)
margin = 8
imin = max(0, imin - margin)
jmin = max(0, jmin - margin)
imax = min(nx_total - 1, imax + margin)
jmax = min(ny_total - 1, jmax + margin)
sub_lons2d = lons2d[imin:imax + 1, jmin:jmax + 1]
sub_lats2d = lats2d[imin:imax + 1, jmin:jmax + 1]
sub_flow_directions = manager.flow_directions[imin:imax + 1, jmin:jmax + 1]
sub_flow_in_mask = flow_in_mask[imin:imax + 1, jmin:jmax + 1]
sub_i_upstream, sub_j_upstream = np.where(sub_flow_in_mask == 1)
basemap = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=sub_lons2d, lats2d=sub_lats2d, resolution="h"
)
#plot all stations
#stations = sim_name_to_station_to_model_point.items()[0][1].keys()
x_list = [the_station.longitude for the_station in (s,)]
y_list = [the_station.latitude for the_station in (s,)]
basemap_big = Crcm5ModelDataManager.get_rotpole_basemap_using_lons_lats(
lons2d=lons2d, lats2d=lats2d
)
x_list, y_list = basemap_big(x_list, y_list)
basemap_big.scatter(x_list, y_list, c="r", s=40, linewidths=0, ax=ax)
basemap_big.drawcoastlines(ax=ax)
basemap_big.drawrivers(ax=ax)
ax.annotate(s.id, xy=basemap(s.longitude, s.latitude), xytext=(3, 3), textcoords='offset points',
font_properties=FontProperties(weight="bold"), bbox=dict(facecolor='w', alpha=1),
ha="left", va="bottom", zorder=2)
x_big, y_big = basemap_big(manager.lons2D, manager.lats2D)
####zoom to the area of interest
#axins = zoomed_inset_axes(ax, 3, loc=2)
displayCoords = ax.transData.transform((x_big[imin, jmin], y_big[imin, jmin]))
x_shift_fig, y_shift_fig = ax.figure.transFigure.inverted().transform(displayCoords)
print("After transData", displayCoords)
print("xshift and yshift", x_shift_fig, y_shift_fig)
#ax.annotate("yuyu", xy = ( 0.733264985153, 0.477182994408), xycoords = "figure fraction" )
rect = [0.1, y_shift_fig + 0.1, 0.4, 0.4]
axins = ax.figure.add_axes(rect)
#assert isinstance(axins, Axes)
x, y = basemap(sub_lons2d, sub_lats2d)
x1d_start = x[sub_flow_in_mask == 1]
y1d_start = y[sub_flow_in_mask == 1]
fld1d = sub_flow_directions[sub_flow_in_mask == 1]
from util import direction_and_value
ishift, jshift = direction_and_value.flowdir_values_to_shift(fld1d)
sub_i_upstream_next = sub_i_upstream + ishift
sub_j_upstream_next = sub_j_upstream + jshift
u = x[sub_i_upstream_next, sub_j_upstream_next] - x1d_start
v = y[sub_i_upstream_next, sub_j_upstream_next] - y1d_start
u2d = np.ma.masked_all_like(x)
v2d = np.ma.masked_all_like(y)
u2d[sub_i_upstream, sub_j_upstream] = u
v2d[sub_i_upstream, sub_j_upstream] = v
basemap.quiver(x, y, u2d, v2d, angles="xy", scale_units="xy", scale=1, ax=axins)
x_list = [the_station.longitude for the_station in (s,)]
y_list = [the_station.latitude for the_station in (s,)]
x_list, y_list = basemap(x_list, y_list)
basemap.scatter(x_list, y_list, c="r", s=40, linewidths=0)
basemap.drawcoastlines(ax=axins)
basemap.drawrivers(ax=axins)
#read waterbase file, and plot only the polygons which contain at least one upstream point
shp_path = "/skynet3_exec1/huziy/Netbeans Projects/Java/DDM/data/shape/waterbase/na_bas_ll_r500m/na_bas_ll_r500m.shp"
c = fiona.open(shp_path)
hits = c.filter(bbox=(sub_lons2d[0, 0], sub_lats2d[0, 0], sub_lons2d[-1, -1], sub_lats2d[-1, -1]))
points = [Point(the_x, the_y) for the_x, the_y in zip(x1d_start, y1d_start)]
selected_polygons = []
for feature in hits:
new_coords = []
old_coords = feature["geometry"]["coordinates"]
#transform to the map coordinates
for ring in old_coords:
x1 = [tup[0] for tup in ring]
y1 = [tup[1] for tup in ring]
x2, y2 = basemap(x1, y1)
new_coords.append(list(zip(x2, y2)))
feature["geometry"]["coordinates"] = new_coords
poly = shape(feature["geometry"])
#print poly, type(poly)
#print feature.keys()
#print feature["properties"]
prep_poly = prep.prep(poly)
hits = list(filter(prep_poly.contains, points))
if len(hits) > 2:
selected_polygons.append(feature["geometry"])
for p in selected_polygons:
axins.add_patch(PolygonPatch(p, fc="none", ec="b", lw=1.5))
zoom_lines_color = "#6600FF"
#set color of the frame
for child in axins.get_children():
if isinstance(child, Spine):
child.set_color(zoom_lines_color)
child.set_linewidth(3)
# draw a bbox of the region of the inset axes in the parent axes and
# connecting lines between the bbox and the inset axes area
mark_inset(ax, axins, loc1=1, loc2=3, fc="none", ec=zoom_lines_color, lw=3)
#basemap.readshapefile(, "basin")
pass
def handle(self, **options):
shape_file = options['shape']
interpret = options['interpret']
predict = options['predict']
dataset = options['dataset']
year = options['year']
scheme = options['scheme']
filter = options['filter']
filename = basename(shape_file, False)
if filter is not None and filter <= 1:
filter = None
with fiona.open(shape_file) as source:
project = partial(pyproj.transform, pyproj.Proj(source.crs),
pyproj.Proj(init='EPSG:4326'))
if filter == None:
object_list = [(TrainObject(the_geom=GEOSGeometry(
transform(project, shape(feat['geometry'])).wkt),
filename=filename,
creation_year=year),
feat['properties']) for feat in source]
else:
object_list = [
(TrainObject(the_geom=GEOSGeometry(
transform(project, shape(feat['geometry'])).wkt),
filename=filename,
creation_year=year), feat['properties'])
for ind, feat in enumerate(source) if ind % filter == 0
]
TrainObject.objects.bulk_create(list(map(lambda x: x[0], object_list)))
train_classification_objects = []
tag_map = {}
for tup in object_list:
o = tup[0]
p = tup[1]
value = p[interpret]
interpret_tag = tag_map.get(value)
if not interpret_tag:
interpret_tag, created = Tag.objects.get_or_create(
numeric_code=value, scheme=scheme)
tag_map[value] = interpret_tag
if created:
logger.info(
'New tag created with values: numeric_code=%s, scheme=%s'
% (value, scheme))
predict_tag = None
if predict is not None:
value = p[predict]
predict_tag = tag_map.get(value)
if not predict_tag:
try:
predict_tag = Tag.objects.get(numeric_code=value,
scheme=scheme)
except Tag.DoesNotExist:
predict_tag = Tag.objects.get(numeric_code=value,
scheme=scheme)
predict_tag.save()
tag_map[value] = predict_tag
train_classification_objects.append(
TrainClassification(train_object=o,
predict_tag=predict_tag,
interpret_tag=interpret_tag,
training_set=dataset))
TrainClassification.objects.bulk_create(train_classification_objects)