def decode_event_path(self, event_path):
""" decodes an event path, with
chrono_path tiles as the first
two characters of each level, with
the geo tile as the last character
"""
chrono_prefix = ''
if self.CHRONO_PREFIX_EVENT_SEP in event_path:
event_ex = event_path.split(self.CHRONO_PREFIX_EVENT_SEP)
chrono_prefix = event_ex[0]
event_path = event_ex[1]
event_path_list = event_path.split(self.LEVEL_DELIM)
chrono_path = ''
geo_path = ''
for event_tile in event_path_list:
chrono_path = self.add_to_path(event_tile[0],
chrono_path,
self.CHRONO_TILE_DEPTH)
chrono_path = self.add_to_path(event_tile[1],
chrono_path,
self.CHRONO_TILE_DEPTH)
geo_path = self.add_to_path(event_tile[2],
geo_path,
self.GEO_TILE_DEPTH)
chrono_path = chrono_prefix + chrono_path
ch = ChronoTile()
gm = GlobalMercator()
output = {'chrono_path': chrono_path,
'chrono': ch.decode_path_dates(chrono_path),
'geo_path': geo_path,
'geo': gm.quadtree_to_lat_lon(geo_path)}
return output
def make_geotile_filter_label(raw_geotile):
"""Parses a raw bbox parameter value to make a filter label
"""
if configs.REQUEST_OR_OPERATOR in raw_geotile:
tile_list = raw_geotile.split(configs.REQUEST_OR_OPERATOR)
else:
tile_list = [raw_geotile]
output_list = []
for tile in tile_list:
geotile = GlobalMercator()
coordinates = geotile.quadtree_to_lat_lon(tile)
if not coordinates:
label = '[Ignored invalid geospatial tile]'
else:
round_level = utilities.estimate_good_coordinate_rounding(
lon_a=coordinates[0],
lat_a=coordinates[1],
lon_b=coordinates[2],
lat_b=coordinates[3],
)
label = 'In the region bounded by: {}, {} (SW) and {}, {} (NE)'.format(
round(coordinates[0], round_level),
round(coordinates[1], round_level),
round(coordinates[2], round_level),
round(coordinates[3], round_level),
)
output_list.append(label)
output = '; or '.join(output_list)
return output
def _make_valid_options_tile_tuples(self, options_tuples):
"""Makes a list of valid tile_dicts from a list of options_tuples"""
valid_tile_tuples = []
for tile, count in options_tuples:
if tile.startswith(NULL_ISLAND_TILE_ROOT):
# Skip, because the tile is almost certainly reflecting
# bad data.
logger.warn('Found {} bad geotile records'.format(count))
# So skip.
continue
# Parse the tile to get the lon, lat list.
gm = GlobalMercator()
geo_coords = gm.quadtree_to_geojson_poly_coords(tile)
if not isinstance(geo_coords, list):
# Not a valid data for some awful reason.
logger.warn('Found bad {} geotile with {} records'.format(
tile,
count,
))
continue
valid_tile_tuples.append((
tile,
count,
))
return valid_tile_tuples
def decode_event_path(self, event_path):
""" decodes an event path, with
chrono_path tiles as the first
two characters of each level, with
the geo tile as the last character
"""
chrono_prefix = ''
if self.CHRONO_PREFIX_EVENT_SEP in event_path:
event_ex = event_path.split(self.CHRONO_PREFIX_EVENT_SEP)
chrono_prefix = event_ex[0]
event_path = event_ex[1]
event_path_list = event_path.split(self.LEVEL_DELIM)
chrono_path = ''
geo_path = ''
for event_tile in event_path_list:
chrono_path = self.add_to_path(event_tile[0], chrono_path,
self.CHRONO_TILE_DEPTH)
chrono_path = self.add_to_path(event_tile[1], chrono_path,
self.CHRONO_TILE_DEPTH)
geo_path = self.add_to_path(event_tile[2], geo_path,
self.GEO_TILE_DEPTH)
chrono_path = chrono_prefix + chrono_path
ch = ChronoTile()
gm = GlobalMercator()
output = {
'chrono_path': chrono_path,
'chrono': ch.decode_path_dates(chrono_path),
'geo_path': geo_path,
'geo': gm.quadtree_to_lat_lon(geo_path)
}
return output
def meters_to_lat_lon(request):
""" Converts Web mercator meters to WGS-84 lat / lon """
gm = GlobalMercator()
mx = None
my = None
if request.GET.get('mx') is not None:
mx = request.GET['mx']
if request.GET.get('my') is not None:
my = request.GET['my']
try:
mx = float(mx)
except:
mx = False
try:
my = float(my)
except:
my = False
if isinstance(mx, float) and isinstance(my, float):
lat_lon = gm.MetersToLatLon(mx, my)
output = LastUpdatedOrderedDict()
if len(lat_lon) > 0:
output['lat'] = lat_lon[0]
output['lon'] = lat_lon[1]
else:
output['error'] = 'Stange error, invalid numbers?'
return HttpResponse(json.dumps(output,
ensure_ascii=False,
indent=4),
content_type='application/json; charset=utf8')
else:
return HttpResponse('mx and my paramaters must be numbers',
status=406)
def _process_geo(self):
"""
Finds geospatial point coordinates in GeoJSON features for indexing.
Only 1 location of a given location type is allowed.
"""
self.geo_specified = False
if 'features' in self.oc_item.json_ld:
discovery_done = False
for feature in self.oc_item.json_ld['features']:
ftype = False
loc_type = False
coords = False
try:
ftype = feature['geometry']['type']
except KeyError:
ftype = False
try:
loc_type = feature['properties']['type']
except KeyError:
loc_type = False
try:
zoom = feature['properties']['location-precision']
except KeyError:
zoom = 20
try:
ref_type = feature['properties']['reference-type']
if ref_type == 'specified':
self.geo_specified = True
except KeyError:
ref_type = False
if ftype == 'Point' \
and (loc_type == 'oc-gen:discovey-location'
or loc_type == 'oc-gen:geo-coverage')\
and discovery_done is False:
try:
coords = feature['geometry']['coordinates']
except KeyError:
coords = False
if coords is not False:
gm = GlobalMercator()
self.fields['discovery_geotile'] = \
gm.geojson_coords_to_quadtree(coords, zoom)
# indexing with coordinates seperated by a space is in
# lat-lon order, the reverse of GeoJSON because
# solr spatial fields expect a lat-lon order
lat_ok = self.validate_geo_coordinate(coords[1], 'lat')
lon_ok = self.validate_geo_coordinate(coords[0], 'lon')
if lat_ok and lon_ok:
self.fields['discovery_geolocation'] = \
str(coords[1]) + ',' + str(coords[0])
else:
print('Geo problem in: ' + self.oc_item.uuid + ' ' + str(coords[0]) + ' ' + str(coords[1]))
discovery_done = True # so we don't repeat getting
# discovery locations
if discovery_done:
break
def process_solr_tiles(self, solr_tiles):
""" processes the solr_json
discovery geo tiles,
aggregating to a certain
depth
"""
# first aggregate counts for tile that belong togther
aggregate_tiles = self.aggregate_spatial_tiles(solr_tiles)
# now generate GeoJSON for each tile region
# print('Total tiles: ' + str(t) + ' reduced to ' + str(len(aggregate_tiles)))
i = 0
for tile_key, aggregate_count in aggregate_tiles.items():
i += 1
add_region = True
fl = FilterLinks()
fl.base_request_json = self.filter_request_dict_json
fl.spatial_context = self.spatial_context
new_rparams = fl.add_to_request('disc-geotile',
tile_key)
record = LastUpdatedOrderedDict()
record['id'] = fl.make_request_url(new_rparams)
record['json'] = fl.make_request_url(new_rparams, '.json')
record['count'] = aggregate_count
record['type'] = 'Feature'
record['category'] = 'oc-api:geo-facet'
if self.min_date is not False \
and self.max_date is not False:
when = LastUpdatedOrderedDict()
when['id'] = '#event-' + tile_key
when['type'] = 'oc-gen:formation-use-life'
# convert numeric to GeoJSON-LD ISO 8601
when['start'] = ISOyears().make_iso_from_float(self.min_date)
when['stop'] = ISOyears().make_iso_from_float(self.max_date)
record['when'] = when
gm = GlobalMercator()
geo_coords = gm.quadtree_to_geojson_poly_coords(tile_key)
geometry = LastUpdatedOrderedDict()
geometry['id'] = '#geo-disc-tile-geom-' + tile_key
geometry['type'] = 'Polygon'
geometry['coordinates'] = geo_coords
record['geometry'] = geometry
properties = LastUpdatedOrderedDict()
properties['id'] = '#geo-disc-tile-' + tile_key
properties['href'] = record['id']
properties['label'] = 'Discovery region (' + str(i) + ')'
properties['feature-type'] = 'discovery region (facet)'
properties['count'] = aggregate_count
properties['early bce/ce'] = self.min_date
properties['late bce/ce'] = self.max_date
record['properties'] = properties
if len(tile_key) >= 6:
if tile_key[:6] == '211111':
# no bad coordinates (off 0, 0 coast of Africa)
add_region = False # don't display items without coordinates
if add_region:
self.geojson_regions.append(record)
def process_solr_tiles(self, solr_tiles):
""" processes the solr_json
discovery geo tiles,
aggregating to a certain
depth
"""
# first aggregate counts for tile that belong togther
aggregate_tiles = self.aggregate_spatial_tiles(solr_tiles)
# now generate GeoJSON for each tile region
# print('Total tiles: ' + str(t) + ' reduced to ' + str(len(aggregate_tiles)))
i = 0
for tile_key, aggregate_count in aggregate_tiles.items():
i += 1
add_region = True
fl = FilterLinks()
fl.base_request_json = self.filter_request_dict_json
fl.spatial_context = self.spatial_context
new_rparams = fl.add_to_request('disc-geotile', tile_key)
record = LastUpdatedOrderedDict()
record['id'] = fl.make_request_url(new_rparams)
record['json'] = fl.make_request_url(new_rparams, '.json')
record['count'] = aggregate_count
record['type'] = 'Feature'
record['category'] = 'oc-api:geo-facet'
if self.min_date is not False \
and self.max_date is not False:
when = LastUpdatedOrderedDict()
when['id'] = '#event-' + tile_key
when['type'] = 'oc-gen:formation-use-life'
# convert numeric to GeoJSON-LD ISO 8601
when['start'] = ISOyears().make_iso_from_float(self.min_date)
when['stop'] = ISOyears().make_iso_from_float(self.max_date)
record['when'] = when
gm = GlobalMercator()
geo_coords = gm.quadtree_to_geojson_poly_coords(tile_key)
geometry = LastUpdatedOrderedDict()
geometry['id'] = '#geo-disc-tile-geom-' + tile_key
geometry['type'] = 'Polygon'
geometry['coordinates'] = geo_coords
record['geometry'] = geometry
properties = LastUpdatedOrderedDict()
properties['id'] = '#geo-disc-tile-' + tile_key
properties['href'] = record['id']
properties['label'] = 'Discovery region (' + str(i) + ')'
properties['feature-type'] = 'discovery region (facet)'
properties['count'] = aggregate_count
properties['early bce/ce'] = self.min_date
properties['late bce/ce'] = self.max_date
record['properties'] = properties
if len(tile_key) >= 6:
if tile_key[:6] == '211111':
# no bad coordinates (off 0, 0 coast of Africa)
add_region = False # don't display items without coordinates
if add_region:
self.geojson_regions.append(record)
def _process_geo(self):
"""
Finds geospatial point coordinates in GeoJSON features for indexing.
Only 1 location of a given location type is allowed.
"""
self.geo_specified = False
if "features" in self.oc_item.json_ld:
discovery_done = False
for feature in self.oc_item.json_ld["features"]:
ftype = False
loc_type = False
coords = False
try:
ftype = feature["geometry"]["type"]
except KeyError:
ftype = False
try:
loc_type = feature["properties"]["type"]
except KeyError:
loc_type = False
try:
zoom = feature["properties"]["location-precision"]
except KeyError:
zoom = 20
try:
ref_type = feature["properties"]["reference-type"]
if ref_type == "specified":
self.geo_specified = True
except KeyError:
ref_type = False
if (
ftype == "Point"
and (loc_type == "oc-gen:discovey-location" or loc_type == "oc-gen:geo-coverage")
and discovery_done is False
):
try:
coords = feature["geometry"]["coordinates"]
except KeyError:
coords = False
if coords is not False:
gm = GlobalMercator()
self.fields["discovery_geotile"] = gm.geojson_coords_to_quadtree(coords, zoom)
# indexing with coordinates seperated by a space is in
# lat-lon order, the reverse of GeoJSON because
# solr spatial fields expect a lat-lon order
lat_ok = self.validate_geo_coordinate(coords[1], "lat")
lon_ok = self.validate_geo_coordinate(coords[0], "lon")
if lat_ok and lon_ok:
self.fields["discovery_geolocation"] = str(coords[1]) + "," + str(coords[0])
else:
print("Geo problem in: " + self.oc_item.uuid + " " + str(coords[0]) + " " + str(coords[1]))
discovery_done = True # so we don't repeat getting
# discovery locations
if discovery_done:
break
def encode_path_from_geo_chrono(self,
lat,
lon,
latest_bce_ce,
earliest_bce_ce,
chrono_prefix=''):
""" encodes an event path from numeric lat, lon, and date values """
gm = GlobalMercator()
geo_path = gm.lat_lon_to_quadtree(lat, lon)
ch = ChronoTile()
chrono_path = ch.encode_path_from_bce_ce(latest_bce_ce, earliest_bce_ce, chrono_prefix)
return self.encode_path_from_geo_chrono_paths(geo_path, chrono_path)
def make_geo_meta(self, project_uuid, sub_projs=False):
output = False
self.project_uuid = project_uuid
if sub_projs is False:
# check if there are subjects in this project
pr = ProjectRels()
sub_projs = pr.get_sub_projects(project_uuid)
if sub_projs is False:
uuids = [project_uuid]
else:
uuids = []
for sub_proj in sub_projs:
uuids.append(sub_proj.uuid)
uuids.append(project_uuid)
self.get_geo_range(uuids)
if self.geo_range is False:
pass
if self.print_progress:
print('Range fail: ' + str(self.geo_range) )
else:
if self.print_progress:
print('Working on range: ' + str(self.geo_range) )
min_lon_lat = [self.geo_range['longitude__min'],
self.geo_range['latitude__min']]
max_lon_lat = [self.geo_range['longitude__max'],
self.geo_range['latitude__max']]
min_point = np.fromiter(min_lon_lat, np.dtype('float'))
max_point = np.fromiter(max_lon_lat, np.dtype('float'))
gm = GlobalMercator()
self.max_geo_range = gm.distance_on_unit_sphere(min_point[1],
min_point[0],
max_point[1],
max_point[0])
if self.print_progress:
print('Max geo range: ' + str(self.max_geo_range))
if self.max_geo_range == 0:
# only 1 geopoint known for the project
proc_centroids = []
proc_centroid = {}
proc_centroid['index'] = 0
proc_centroid['id'] = 1
proc_centroid['num_points'] = 1
proc_centroid['cent_lon'] = self.geo_range['longitude__min']
proc_centroid['cent_lat'] = self.geo_range['latitude__max']
proc_centroid['box'] = False
proc_centroids.append(proc_centroid)
else:
# need to cluster geo data
proc_centroids = self.cluster_geo(uuids)
self.make_geo_objs(proc_centroids)
output = True
return output
def make_geo_meta(self, project_uuid, sub_projs=False):
output = False
self.project_uuid = project_uuid
if sub_projs is False:
# check if there are subjects in this project
pr = ProjectRels()
sub_projs = pr.get_sub_projects(project_uuid)
if sub_projs is False:
uuids = [project_uuid]
else:
uuids = []
for sub_proj in sub_projs:
uuids.append(sub_proj.uuid)
uuids.append(project_uuid)
self.get_geo_range(uuids)
if self.geo_range is False:
pass
if self.print_progress:
print('Range fail: ' + str(self.geo_range) )
else:
if self.print_progress:
print('Working on range: ' + str(self.geo_range) )
min_lon_lat = [self.geo_range['longitude__min'],
self.geo_range['latitude__min']]
max_lon_lat = [self.geo_range['longitude__max'],
self.geo_range['latitude__max']]
min_point = np.fromiter(min_lon_lat, np.dtype('float'))
max_point = np.fromiter(max_lon_lat, np.dtype('float'))
gm = GlobalMercator()
self.max_geo_range = gm.distance_on_unit_sphere(min_point[1],
min_point[0],
max_point[1],
max_point[0])
if self.print_progress:
print('Max geo range: ' + str(self.max_geo_range))
if self.max_geo_range == 0:
# only 1 geopoint known for the project
proc_centroids = []
proc_centroid = {}
proc_centroid['index'] = 0
proc_centroid['id'] = 1
proc_centroid['num_points'] = 1
proc_centroid['cent_lon'] = self.geo_range['longitude__min']
proc_centroid['cent_lat'] = self.geo_range['latitude__max']
proc_centroid['box'] = False
proc_centroids.append(proc_centroid)
else:
# need to cluster geo data
proc_centroids = self.cluster_geo(uuids)
self.make_geo_objs(proc_centroids)
output = True
return output
def encode_path_from_geo_chrono(self,
lat,
lon,
latest_bce_ce,
earliest_bce_ce,
chrono_prefix=''):
""" encodes an event path from numeric lat, lon, and date values """
gm = GlobalMercator()
geo_path = gm.lat_lon_to_quadtree(lat, lon)
ch = ChronoTile()
chrono_path = ch.encode_path_from_bce_ce(latest_bce_ce,
earliest_bce_ce,
chrono_prefix)
return self.encode_path_from_geo_chrono_paths(geo_path, chrono_path)
def make_min_size_region(self, region_dict):
""" widens a coordinate pair based on maximum distance
between points
this makes a square (on a mercator projection) bounding box
region. it will have different real-world distances in the
east west direction between the northern and southern sides
"""
min_distance = self.max_geo_range * 0.05
gm = GlobalMercator()
# measure the north south distance
mid_lat = (region_dict['min_lat'] + region_dict['max_lat']) / 2
mid_lon = (region_dict['min_lon'] + region_dict['max_lon']) / 2
ns_diag_dist = gm.distance_on_unit_sphere(region_dict['min_lat'],
mid_lon,
region_dict['max_lat'],
mid_lon)
ew_diag_dist = gm.distance_on_unit_sphere(mid_lat,
region_dict['min_lon'],
mid_lat,
region_dict['max_lon'])
if ns_diag_dist < min_distance:
# the north-south distance is too small, so widen it.
# first, find a point south of the mid lat, at the right distance
new_lat_s = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
180)
# second, find a point north of the mid lat, at the right distance
new_lat_n = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
0)
region_dict['min_lat'] = new_lat_s['lat']
region_dict['max_lat'] = new_lat_n['lat']
if ew_diag_dist < min_distance:
# the east-west distance is too small, so widen it.
# first, find a point south of the mid lat, at the right distance
new_lon_w = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
270)
# second, find a point north of the mid lat, at the right distance
new_lon_e = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
90)
region_dict['min_lon'] = new_lon_w['lon']
region_dict['max_lon'] = new_lon_e['lon']
return region_dict
def check_ok_cluster(self, proc_centroid, centroids, uuids):
""" checks to see if the proc_centroid is an OK
cluster, based on the number of items it
contains or if it is far from all other centroids
"""
ok_cluster = True # default to this being a good cluster
if proc_centroid['num_points'] < 2:
# the cluster has only 1 point, meaning it may be too small
db_multiple = self.check_ok_cluster_for_lone_point(uuids,
proc_centroid['max_lon'],
proc_centroid['max_lat'])
if db_multiple is False:
# not many records,
# OK now check if it is far from other points
single_far = True
for o_centroid in centroids:
o_lon = o_centroid[0]
o_lat = o_centroid[1]
if o_lon != proc_centroid['cent_lon'] \
and o_lat != proc_centroid['cent_lat']:
# not the same centroid, so check distance
gm = GlobalMercator()
cent_dist = gm.distance_on_unit_sphere(o_lat,
o_lon,
proc_centroid['cent_lat'],
proc_centroid['cent_lon'])
if cent_dist < 1000:
if cent_dist < self.MIN_CLUSTER_SIZE_KM \
or cent_dist < (self.max_geo_range * .1):
# we found a case where this point is close
# to another centroid
single_far = False
if single_far is False:
ok_cluster = False
if self.print_progress and ok_cluster is False:
message = 'Cluster Loop: ' + str(proc_centroid['cluster_loop']) + ', cluster: ' + str(proc_centroid['index']) + ' '
message += ' has few items, too close with other centroids.'
print(message)
return ok_cluster
def lat_lon_to_quadtree(request):
""" Converts WGS-84 lat / lon to a quadtree tile of a given zoom level """
gm = GlobalMercator()
lat = None
lon = None
rand = None
lat_ok = False
lon_ok = False
zoom = gm.MAX_ZOOM
if request.GET.get('lat') is not None:
lat = request.GET['lat']
lat_ok = gm.validate_geo_coordinate(lat, 'lat')
if request.GET.get('lon') is not None:
lon = request.GET['lon']
lon_ok = gm.validate_geo_coordinate(lon, 'lon')
if request.GET.get('zoom') is not None:
check_zoom = request.GET['zoom']
dtc_obj = DescriptionDataType()
zoom = dtc_obj.validate_integer(check_zoom)
if zoom is not None:
# zoom is valid
if zoom > gm.MAX_ZOOM:
zoom = gm.MAX_ZOOM
elif zoom < 1:
zoom = 1
if request.GET.get('rand') is not None:
dtc_obj = DescriptionDataType()
rand = dtc_obj.validate_numeric(request.GET['rand'])
if lat_ok and lon_ok and zoom is not None:
output = gm.lat_lon_to_quadtree(lat, lon, zoom)
return HttpResponse(output,
content_type='text/plain; charset=utf8')
else:
message = 'ERROR: "lat" and "lon" parameters must be valid WGS-84 decimal degrees'
if zoom is None:
message += ', "zoom" parameter needs to be an integer between 1 and ' + str(gm.MAX_ZOOM) + '.'
return HttpResponse(message,
content_type='text/plain; charset=utf8',
status=406)
def check_ok_cluster(self, proc_centroid, centroids, uuids):
""" checks to see if the proc_centroid is an OK
cluster, based on the number of items it
contains or if it is far from all other centroids
"""
ok_cluster = True # default to this being a good cluster
if proc_centroid['num_points'] < 2:
# the cluster has only 1 point, meaning it may be too small
db_multiple = self.check_ok_cluster_for_lone_point(uuids,
proc_centroid['max_lon'],
proc_centroid['max_lat'])
if db_multiple is False:
# not many records,
# OK now check if it is far from other points
single_far = True
for o_centroid in centroids:
o_lon = o_centroid[0]
o_lat = o_centroid[1]
if o_lon != proc_centroid['cent_lon'] \
and o_lat != proc_centroid['cent_lat']:
# not the same centroid, so check distance
gm = GlobalMercator()
cent_dist = gm.distance_on_unit_sphere(o_lat,
o_lon,
proc_centroid['cent_lat'],
proc_centroid['cent_lon'])
if cent_dist < 1000:
if cent_dist < self.MIN_CLUSTER_SIZE_KM \
or cent_dist < (self.max_geo_range * .1):
# we found a case where this point is close
# to another centroid
single_far = False
if single_far is False:
ok_cluster = False
if self.print_progress and ok_cluster is False:
message = 'Cluster Loop: ' + str(proc_centroid['cluster_loop']) + ', cluster: ' + str(proc_centroid['index']) + ' '
message += ' has few items, too close with other centroids.'
print(message)
return ok_cluster
def quadtree_to_lat_lon(request):
""" Converts a quadtree tile to WGS-84 lat / lon coordinates in different formats """
lat_lon = None
gm = GlobalMercator()
if request.GET.get('tile') is not None:
tile = request.GET['tile']
try:
lat_lon = gm.quadtree_to_lat_lon(tile)
except:
lat_lon = None
if lat_lon is not None:
# default to json format with lat, lon dictionary object
lat_lon_dict = LastUpdatedOrderedDict()
lat_lon_dict['lat'] = lat_lon[0]
lat_lon_dict['lon'] = lat_lon[1]
output = json.dumps(lat_lon_dict,
ensure_ascii=False,
indent=4)
content_type='application/json; charset=utf8'
if request.GET.get('format') is not None:
# client requested another format, give it if recognized
if request.GET['format'] == 'geojson':
# geojson format, with longitude then latitude
output = json.dumps(([lat_lon[1], lat_lon[0]]),
ensure_ascii=False,
indent=4)
content_type='application/json; charset=utf8'
elif request.GET['format'] == 'lat,lon':
# text format, with lat, comma lon coordinates
output = str(lat_lon[0]) + ',' + str(lat_lon[1])
content_type='text/plain; charset=utf8'
return HttpResponse(output,
content_type=content_type)
else:
message = 'ERROR: "tile" must be a valid quadtree geospatial tile (string composed of digits ranging from 0-3)'
return HttpResponse(message,
content_type='text/plain; charset=utf8',
status=406)
def _distance_determine_aggregation_depth(self, valid_tile_tuples):
"""Uses distance between to recalibrate aggregation depth in tiles"""
if len(valid_tile_tuples) < 2:
return self.default_aggregation_depth
lons = []
lats = []
gm = GlobalMercator()
for tile, _ in valid_tile_tuples:
geo_coords = gm.quadtree_to_geojson_lon_lat(tile)
# Remember geojson ordering of the coordinates (lon, lat)
lons.append(geo_coords[0])
lats.append(geo_coords[1])
max_distance = gm.distance_on_unit_sphere(
min(lats),
min(lons),
max(lats),
max(lons),
)
# Converts the maximum distance between points into a zoom level
# appropriate for tile aggregation. Seems to work well.
return gm.ZoomForPixelSize(max_distance) + 3
def make_geotile_filter_label(self, raw_geotile):
""" parses a raw bbox parameter value to make
a filter label
"""
output_list = []
if '||' in raw_geotile:
tile_list = raw_geotile.split('||')
else:
tile_list = [raw_geotile]
for tile in tile_list:
geotile = GlobalMercator()
coordinates = geotile.quadtree_to_lat_lon(tile)
if coordinates is not False:
label = 'In the region bounded by: '
label += str(round(coordinates[0], 3))
label += ', ' + str(round(coordinates[1], 3))
label += ' (SW) and ' + str(round(coordinates[2], 3))
label += ', ' + str(round(coordinates[3], 3))
label += ' (NE)'
output_list.append(label)
else:
output_list.append('[Ignored invalid geospatial tile]')
output = '; or '.join(output_list)
return output
def make_min_size_region(self, region_dict):
""" widens a coordinate pair based on maximum distance
between points
this makes a square (on a mercator projection) bounding box
region. it will have different real-world distances in the
east west direction between the northern and southern sides
"""
min_distance = self.max_geo_range * 0.05
gm = GlobalMercator()
# measure the north south distance
mid_lat = (region_dict['min_lat'] + region_dict['max_lat']) / 2
mid_lon = (region_dict['min_lon'] + region_dict['max_lon']) / 2
ns_diag_dist = gm.distance_on_unit_sphere(region_dict['min_lat'],
mid_lon,
region_dict['max_lat'],
mid_lon)
ew_diag_dist = gm.distance_on_unit_sphere(mid_lat,
region_dict['min_lon'],
mid_lat,
region_dict['max_lon'])
if ns_diag_dist < min_distance:
# the north-south distance is too small, so widen it.
# first, find a point south of the mid lat, at the right distance
new_lat_s = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
180)
# second, find a point north of the mid lat, at the right distance
new_lat_n = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
0)
region_dict['min_lat'] = new_lat_s['lat']
region_dict['max_lat'] = new_lat_n['lat']
if ew_diag_dist < min_distance:
# the east-west distance is too small, so widen it.
# first, find a point south of the mid lat, at the right distance
new_lon_w = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
270)
# second, find a point north of the mid lat, at the right distance
new_lon_e = gm.get_point_by_distance_from_point(mid_lat,
mid_lon,
(min_distance / 2),
90)
region_dict['min_lon'] = new_lon_w['lon']
region_dict['max_lon'] = new_lon_e['lon']
return region_dict
def get_geotile_scope(self, solr_tiles):
""" find the most specific tile shared by the whole dataset """
geo_tiles = []
bound_list = []
max_distance = 0
lat_lon_min_max = [{
'min': None,
'max': None
}, {
'min': None,
'max': None
}]
for tile_key in solr_tiles[::2]:
if tile_key[:6] != '211111':
# a bit of a hack to exclude display of
# erroroneous data without spatial reference
geo_tiles.append(tile_key)
gm = GlobalMercator()
bounds = gm.quadtree_to_lat_lon(tile_key)
lat_lon_min_max = self.get_min_max(lat_lon_min_max, bounds)
bound_list.append(bounds)
if len(geo_tiles) > 0:
test_tile = geo_tiles[0] # we compare against the first tile
tile_len = len(test_tile) # size of the tile
in_all_geotiles = True
i = 0
while (i < tile_len and in_all_geotiles):
if i > 0:
test_val = str(test_tile[0:i])
else:
test_val = str(test_tile[0])
for geo_tile in geo_tiles:
if i > len(geo_tile):
in_all_geotiles = False
else:
if i > 0:
geo_tile_part = geo_tile[0:i]
else:
geo_tile_part = geo_tile[0]
if test_val != geo_tile_part:
in_all_geotiles = False
if in_all_geotiles:
# ok! we have somthing that is still in all tiles
self.geotile_scope = test_val
i += 1
if isinstance(self.geotile_scope, str):
self.aggregation_depth += round(len(self.geotile_scope) * 1, 0)
self.aggregation_depth = int(self.aggregation_depth)
if self.aggregation_depth < 6:
self.aggregation_depth = 6
if self.aggregation_depth >= 6 and len(bound_list) >= 2:
gm = GlobalMercator()
max_distance = gm.distance_on_unit_sphere(
lat_lon_min_max[0]['min'], lat_lon_min_max[1]['min'],
lat_lon_min_max[0]['max'], lat_lon_min_max[1]['max'])
if max_distance == 0:
self.aggregation_depth = 10
else:
# converts the maximum distance between points into a zoom level
# appropriate for tile aggregation. seems to work well.
self.aggregation_depth = gm.ZoomForPixelSize(max_distance) + 3
# print('now: ' + str(self.aggregation_depth) + ' for ' + str(max_distance))
if self.aggregation_depth > self.max_depth:
self.aggregation_depth = self.max_depth
if self.aggregation_depth < 6:
self.aggregation_depth = 6
return self.geotile_scope
def __init__(self):
self.CHRONO_TILE_DEPTH = ChronoTile().MAX_TILE_DEPTH
self.GEO_TILE_DEPTH = GlobalMercator().MAX_ZOOM
self.LEVEL_DELIM = '-' # so as not to get confused with the chrono-path '-' prefix
self.CHRONO_PREFIX_EVENT_SEP = 'e-'
def make_geotile_facet_options(self, solr_json):
"""Makes geographic tile facets from a solr_json response"""
geotile_path_keys = (configs.FACETS_SOLR_ROOT_PATH_KEYS +
['discovery_geotile'])
geotile_val_count_list = utilities.get_dict_path_value(
geotile_path_keys, solr_json, default=[])
if not len(geotile_val_count_list):
return None
# Make the list of tile, count tuples.
options_tuples = utilities.get_facet_value_count_tuples(
geotile_val_count_list)
if not len(options_tuples):
return None
valid_tile_tuples = self._make_valid_options_tile_tuples(
options_tuples)
if not len(valid_tile_tuples):
# None of the chronological tiles are valid
# given the query requirements.
return None
# Determine the aggregation depth needed to group geotiles
# together into a reasonable number of options.
self._get_tile_aggregation_depth(valid_tile_tuples)
# Determine the min tile depth. We need to return this to
# the client so the client knows not to over-zoom.
tile_lens = [len(tile) for tile, _ in valid_tile_tuples]
self.min_depth = min(tile_lens)
# Get the client's requested feature type for the geotile
# facets.
feature_type = utilities.get_request_param_value(
self.request_dict,
param='geo-facet-type',
default=self.default_tile_feature_type,
as_list=False,
solr_escape=False,
)
if feature_type not in self.valid_tile_feature_types:
# If the requested feature type is not in the
# valid list of feature types, just use the default.
feature_type = self.default_tile_feature_type
aggregate_tiles = {}
for tile, count in valid_tile_tuples:
# Now aggregate the tiles.
trim_tile_key = tile[:self.default_aggregation_depth]
if trim_tile_key not in aggregate_tiles:
# Make the aggregate tile with a count
# of zero
aggregate_tiles[trim_tile_key] = 0
aggregate_tiles[trim_tile_key] += count
options = []
for tile, count in aggregate_tiles.items():
sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict),
base_search_url=self.base_search_url)
# Remove non search related params.
sl.remove_non_query_params()
# Update the request dict for this facet option.
sl.replace_param_value(
'disc-geotile',
match_old_value=None,
new_value=tile,
)
urls = sl.make_urls_from_request_dict()
if urls['html'] == self.current_filters_url:
# The new URL matches our current filter
# url, so don't add this facet option.
continue
option = LastUpdatedOrderedDict()
option['id'] = urls['html']
option['json'] = urls['json']
option['count'] = count
option['type'] = 'Feature'
option['category'] = 'oc-api:geo-facet'
# Add some general chronology information to the
# geospatial tile.
option = self._add_when_object_to_feature_option(
tile,
option,
)
gm = GlobalMercator()
if feature_type == 'Polygon':
# Get polygon coordinates (a list of lists)
geo_coords = gm.quadtree_to_geojson_poly_coords(tile)
elif feature_type == 'Point':
# Get point coordinates (a list of lon,lat values)
geo_coords = gm.quadtree_to_geojson_lon_lat(tile)
else:
# We shouldn't be here!
continue
# Add the geometry object to the facet option.
geometry = LastUpdatedOrderedDict()
geometry['id'] = '#geo-disc-tile-geom-{}'.format(tile)
geometry['type'] = feature_type
geometry['coordinates'] = geo_coords
option['geometry'] = geometry
properties = LastUpdatedOrderedDict()
properties['id'] = '#geo-disc-tile-{}'.format(tile)
properties['href'] = option['id']
properties['label'] = 'Discovery region ({})'.format(
(len(options) + 1))
properties['feature-type'] = 'discovery region (facet)'
properties['count'] = count
properties['early bce/ce'] = self.min_date
properties['late bce/ce'] = self.max_date
option['properties'] = properties
options.append(option)
return options
def process_geo(self):
""" processes the solr_json
discovery geo tiles,
aggregating to a certain
depth
"""
if isinstance(self.geo_pivot, list):
i = 0
for proj in self.geo_pivot:
i += 1
add_feature = True
project_key = proj['value']
proj_ex = project_key.split('___')
slug = proj_ex[0]
uri = self.make_url_from_val_string(proj_ex[2])
href = self.make_url_from_val_string(proj_ex[2], False)
label = proj_ex[3]
fl = FilterLinks()
fl.base_request_json = self.filter_request_dict_json
fl.spatial_context = self.spatial_context
new_rparams = fl.add_to_request('proj', slug)
if 'response' in new_rparams:
new_rparams.pop('response', None)
record = LastUpdatedOrderedDict()
record['id'] = fl.make_request_url(new_rparams)
record['json'] = fl.make_request_url(new_rparams, '.json')
record['count'] = proj['count']
record['type'] = 'Feature'
record['category'] = 'oc-api:geo-project'
min_date = False
max_date = False
if project_key in self.projects:
min_date = self.projects[project_key]['min_date']
max_date = self.projects[project_key]['max_date']
if min_date is not False \
and max_date is not False:
when = LastUpdatedOrderedDict()
when['id'] = '#event-' + slug
when['type'] = 'oc-gen:formation-use-life'
# convert numeric to GeoJSON-LD ISO 8601
when['start'] = ISOyears().make_iso_from_float(
self.projects[project_key]['min_date'])
when['stop'] = ISOyears().make_iso_from_float(
self.projects[project_key]['max_date'])
record['when'] = when
if 'pivot' not in proj:
add_feature = False
else:
geometry = LastUpdatedOrderedDict()
geometry['id'] = '#geo-geom-' + slug
geometry['type'] = 'Point'
pivot_count_total = 0
total_lon = 0
total_lat = 0
for geo_data in proj['pivot']:
pivot_count_total += geo_data['count']
gm = GlobalMercator()
bounds = gm.quadtree_to_lat_lon(geo_data['value'])
mean_lon = (bounds[1] + bounds[3]) / 2
mean_lat = (bounds[0] + bounds[2]) / 2
total_lon += mean_lon * geo_data['count']
total_lat += mean_lat * geo_data['count']
weighted_mean_lon = total_lon / pivot_count_total
weighted_mean_lat = total_lat / pivot_count_total
geometry['coordinates'] = [
weighted_mean_lon, weighted_mean_lat
]
record['geometry'] = geometry
# now make a link to search records for this project
fl = FilterLinks()
fl.spatial_context = self.spatial_context
new_rparams = fl.add_to_request('proj', slug)
search_link = fl.make_request_url(new_rparams)
properties = LastUpdatedOrderedDict()
properties['id'] = '#geo-proj-' + slug
properties['uri'] = uri
properties['href'] = href
properties['search'] = search_link
properties['label'] = label
properties['feature-type'] = 'project '
properties['count'] = proj['count']
properties['early bce/ce'] = min_date
properties['late bce/ce'] = max_date
record['properties'] = properties
if add_feature:
self.geojson_projects.append(record)
def add_geojson(self, json_ld):
"""
adds geospatial and event data that links time and space information
"""
uuid = self.manifest.uuid
item_type = self.manifest.item_type
geo_meta = self.geo_meta
event_meta = self.event_meta
features_dict = False # dict of all features to be added
feature_events = False # mappings between features and time periods
if geo_meta is not False:
# print('here!' + str(geo_meta))
features_dict = LastUpdatedOrderedDict()
feature_events = LastUpdatedOrderedDict()
for geo in geo_meta:
geo_id = geo.feature_id
geo_node = '#geo-' + str(
geo_id) # the node id for database rec of the feature
geo_node_geom = '#geo-geom-' + str(geo_id)
geo_node_props = '#geo-props-' + str(geo_id)
geo_node_derived = '#geo-derived-' + str(
geo_id) # node id for a derived feature
geo_node_derived_geom = '#geo-derived-geom-' + str(geo_id)
geo_node_derived_props = '#geo-derived-props-' + str(geo_id)
feature_events[geo_node] = []
geo_props = LastUpdatedOrderedDict()
geo_props['href'] = URImanagement.make_oc_uri(
uuid, item_type, self.cannonical_uris)
geo_props['type'] = geo.meta_type
if len(geo.note) > 0:
geo_props['note'] = geo.note
if uuid != geo.uuid:
geo_props['reference-type'] = 'inferred'
geo_props['reference-uri'] = URImanagement.make_oc_uri(
geo.uuid, 'subjects', self.cannonical_uris)
rel_meta = self.item_gen_cache.get_entity(geo.uuid)
if rel_meta is not False:
geo_props['reference-label'] = rel_meta.label
geo_props['reference-slug'] = rel_meta.slug
else:
geo_props['reference-label'] = self.manifest.label
geo_props['reference-type'] = 'specified'
if self.assertion_hashes:
geo_props['hash_id'] = geo.hash_id
geo_props['feature_id'] = geo.feature_id
if geo.specificity < 0 and self.manifest.item_type != 'projects':
# case where we've got reduced precision geospatial data
# geotile = quadtree.encode(geo.latitude, geo.longitude, abs(geo.specificity))
geo_props['location-precision'] = abs(geo.specificity)
geo_props[
'location-precision-note'] = 'Location data approximated as a security precaution.'
gmt = GlobalMercator()
geotile = gmt.lat_lon_to_quadtree(geo.latitude,
geo.longitude,
abs(geo.specificity))
tile_bounds = gmt.quadtree_to_lat_lon(geotile)
item_polygon = Polygon([[(tile_bounds[1], tile_bounds[0]),
(tile_bounds[1], tile_bounds[2]),
(tile_bounds[3], tile_bounds[2]),
(tile_bounds[3], tile_bounds[0]),
(tile_bounds[1], tile_bounds[0])]
])
item_f_poly = Feature(geometry=item_polygon)
item_f_poly.id = geo_node_derived
item_f_poly.geometry.id = geo_node_derived_geom
item_f_poly.properties.update(geo_props)
item_f_poly.properties['location-note'] = 'This region defines the '\
'approximate location for this item.'
item_f_poly.properties['id'] = geo_node_derived_props
features_dict[geo_node_derived] = item_f_poly
item_point = Point(
(float(geo.longitude), float(geo.latitude)))
item_f_point = Feature(geometry=item_point)
item_f_point.id = geo_node
item_f_point.geometry.id = geo_node_geom
item_f_point.properties.update(geo_props)
item_f_point.properties['location-note'] = 'This point defines the center of the '\
'region approximating the location for this item.'
item_f_point.properties['id'] = geo_node_props
features_dict[geo_node] = item_f_point
elif len(geo.coordinates) > 1:
# here we have geo_json expressed features and geometries to use
if geo.specificity < 0:
geo_props[
'location-precision-note'] = 'Location data approximated as a security precaution.'
elif geo.specificity > 0:
geo_props[
'location-precision-note'] = 'Location data has uncertainty.'
else:
geo_props['location-precision-note'] = 'Location data available with no '\
'intentional reduction in precision.'
item_point = Point(
(float(geo.longitude), float(geo.latitude)))
item_f_point = Feature(geometry=item_point)
item_f_point.properties.update(geo_props)
if uuid == geo.uuid:
#the item itself has the polygon as it's feature
item_db = Point(
(float(geo.longitude), float(geo.latitude)))
if geo.ftype == 'Polygon':
coord_obj = json.loads(geo.coordinates)
item_db = Polygon(coord_obj)
elif (geo.ftype == 'MultiPolygon'):
coord_obj = json.loads(geo.coordinates)
item_db = MultiPolygon(coord_obj)
elif (geo.ftype == 'MultiLineString'):
coord_obj = json.loads(geo.coordinates)
item_db = MultiLineString(coord_obj)
item_f_db = Feature(geometry=item_db)
item_f_db.id = geo_node
item_f_db.geometry.id = geo_node_geom
item_f_db.properties.update(geo_props)
item_f_db.properties['id'] = geo_node_props
features_dict[geo_node] = item_f_db
item_f_point.id = geo_node_derived
item_f_point.geometry.id = geo_node_derived_geom
item_f_point.properties['location-region-note'] = 'This point represents the center of the '\
'region defining the location of this item.'
item_f_point.properties['id'] = geo_node_derived_props
features_dict[geo_node_derived] = item_f_point
else:
#the item is contained within another item with a polygon or multipolygon feature
item_f_point.id = geo_node
item_f_point.geometry.id = geo_node_geom
item_f_point.properties['id'] = geo_node_props
item_f_point.properties['contained-in-region'] = True
item_f_point.properties['location-region-note'] = 'This point represents the center of the '\
'region containing this item.'
features_dict[geo_node] = item_f_point
else:
# case where the item only has a point for geo-spatial reference
geo_props[
'location-note'] = 'Location data available with no intentional reduction in precision.'
item_point = Point(
(float(geo.longitude), float(geo.latitude)))
item_f_point = Feature(geometry=item_point)
item_f_point.id = geo_node
item_f_point.geometry.id = geo_node_geom
item_f_point.properties.update(geo_props)
item_f_point.properties['id'] = geo_node_props
features_dict[geo_node] = item_f_point
if event_meta is not False:
# events provide chrological information, tied to geo features
# sometimes there are more than 1 time period for each geo feature
# in such cases, we duplicate geo features and add the different time event
# information to the new features
for event in event_meta:
rel_feature_num = 1 # default to the first geospatial feature for where the event happened
rel_feature_node = False
if event.feature_id > 0:
rel_feature_num = event.feature_id
if rel_feature_num >= 1:
rel_feature_node = '#geo-' + str(rel_feature_num)
act_event_obj = LastUpdatedOrderedDict()
act_event_obj = self.add_when_json(act_event_obj, uuid,
item_type, event)
if rel_feature_node is not False and feature_events is not False:
feature_events[rel_feature_node].append(act_event_obj)
if features_dict is not False:
if feature_events is not False:
for node_key, event_list in feature_events.items():
# update the feature with the first event "when" information
if len(event_list) > 0:
features_dict[node_key].update(event_list[0])
event_i = 1
for event in event_list:
if event_i <= 1:
# add the time info to the feature
old_feature = features_dict[node_key]
old_geo_id = old_feature.geometry['id']
old_prop_id = old_feature.properties['id']
features_dict[node_key].update(event)
else:
act_feature = copy.deepcopy(old_feature)
# now add new node ids for the new features created to for the event
new_node = node_key + '-event-' + str(
event_i)
act_feature.id = new_node
act_feature.geometry[
'id'] = old_geo_id + '-event-' + str(
event_i)
act_feature.properties[
'id'] = old_prop_id + '-event-' + str(
event_i)
act_feature.update(
event
) # add the time info to the new feature
features_dict[new_node] = act_feature
del (act_feature)
event_i += 1
feature_keys = list(features_dict.keys())
if len(feature_keys) < 1:
del features_dict[feature_keys[0]][
'id'] # remove the conflicting id
# only 1 feature, so item is not a feature collection
json_ld.update(features_dict[feature_keys[0]])
else:
feature_list = [
] # multiple features, so item has a feature collection
for node_key, feature in features_dict.items():
feature_list.append(feature)
item_fc = FeatureCollection(feature_list)
json_ld.update(item_fc)
return json_ld
def process_geo(self):
""" processes the solr_json
discovery geo tiles,
aggregating to a certain
depth
"""
if isinstance(self.geo_pivot, list):
i = 0
for proj in self.geo_pivot:
i += 1
add_feature = True
project_key = proj['value']
proj_ex = project_key.split('___')
slug = proj_ex[0]
uri = self.make_url_from_val_string(proj_ex[2])
href = self.make_url_from_val_string(proj_ex[2], False)
label = proj_ex[3]
fl = FilterLinks()
fl.base_request_json = self.filter_request_dict_json
fl.spatial_context = self.spatial_context
new_rparams = fl.add_to_request('proj',
slug)
if 'response' in new_rparams:
new_rparams.pop('response', None)
record = LastUpdatedOrderedDict()
record['id'] = fl.make_request_url(new_rparams)
record['json'] = fl.make_request_url(new_rparams, '.json')
record['count'] = proj['count']
record['type'] = 'Feature'
record['category'] = 'oc-api:geo-project'
min_date = False
max_date = False
if project_key in self.projects:
min_date = self.projects[project_key]['min_date']
max_date = self.projects[project_key]['max_date']
if min_date is not False \
and max_date is not False:
when = LastUpdatedOrderedDict()
when['id'] = '#event-' + slug
when['type'] = 'oc-gen:formation-use-life'
# convert numeric to GeoJSON-LD ISO 8601
when['start'] = ISOyears().make_iso_from_float(self.projects[project_key]['min_date'])
when['stop'] = ISOyears().make_iso_from_float(self.projects[project_key]['max_date'])
record['when'] = when
if 'pivot' not in proj:
add_feature = False
else:
geometry = LastUpdatedOrderedDict()
geometry['id'] = '#geo-geom-' + slug
geometry['type'] = 'Point'
pivot_count_total = 0
total_lon = 0
total_lat = 0
for geo_data in proj['pivot']:
pivot_count_total += geo_data['count']
gm = GlobalMercator()
bounds = gm.quadtree_to_lat_lon(geo_data['value'])
mean_lon = (bounds[1] + bounds[3]) / 2
mean_lat = (bounds[0] + bounds[2]) / 2
total_lon += mean_lon * geo_data['count']
total_lat += mean_lat * geo_data['count']
weighted_mean_lon = total_lon / pivot_count_total
weighted_mean_lat = total_lat / pivot_count_total
geometry['coordinates'] = [weighted_mean_lon,
weighted_mean_lat]
record['geometry'] = geometry
# now make a link to search records for this project
fl = FilterLinks()
fl.spatial_context = self.spatial_context
new_rparams = fl.add_to_request('proj',
slug)
search_link = fl.make_request_url(new_rparams)
properties = LastUpdatedOrderedDict()
properties['id'] = '#geo-proj-' + slug
properties['uri'] = uri
properties['href'] = href
properties['search'] = search_link
properties['label'] = label
properties['feature-type'] = 'project '
properties['count'] = proj['count']
properties['early bce/ce'] = min_date
properties['late bce/ce'] = max_date
record['properties'] = properties
if add_feature:
self.geojson_projects.append(record)
