def add_non_point_geojson_coordinates(self, uuid_geo_dict): """Adds non-point geojson coordinates to the result""" geo_obj = uuid_geo_dict.get(self.uuid) if not geo_obj: return None self.geo_feature_type = geo_obj.ftype coord_obj = json.loads(geo_obj.coordinates) v_geojson = ValidateGeoJson() coord_obj = v_geojson.fix_geometry_rings_dir(geo_obj.ftype, coord_obj) self.geometry_coords = coord_obj
def add_feature_to_existing_items(self, feature): """ Process a feature to extract geospatial object. It will: (1) Find the appropriate item in the manifest table (2) Adds a record in the geospace table """ man_obj = None if 'properties' in feature: props = feature['properties'] if 'uri' in props: try_uuid = props['uri'].split('/')[-1] man_objs = Manifest.objects.filter(uuid=try_uuid)[:1] if man_objs: man_obj = man_objs[0] if man_obj and 'geometry' in feature: # first get and validate the coordinates from the GeoJSON file if man_obj.uuid not in self.uuid_counts: self.uuid_counts[man_obj.uuid] = 0 self.uuid_counts[man_obj.uuid] += 1 geometry_type = feature['geometry']['type'] coordinates = feature['geometry']['coordinates'] v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates( geometry_type, coordinates) if not c_ok: print('Fixing coordinates for: {}'.format(man_obj.uuid)) coordinates = v_geojson.fix_geometry_rings_dir( geometry_type, coordinates) if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2: Geospace.objects.filter(uuid=man_obj.uuid).delete() coord_str = json.dumps(coordinates, indent=4, ensure_ascii=False) gg = GeospaceGeneration() lon_lat = gg.get_centroid_lonlat_coordinates( coord_str, geometry_type) print('Saving new geomettry for: ' + str(man_obj.uuid)) geo = Geospace() geo.uuid = man_obj.uuid geo.project_uuid = man_obj.project_uuid geo.source_id = self.source_id geo.item_type = man_obj.item_type geo.feature_id = self.uuid_counts[man_obj.uuid] geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION geo.ftype = geometry_type geo.latitude = lon_lat[1] geo.longitude = lon_lat[0] geo.specificity = 0 # dump coordinates as json string geo.coordinates = coord_str try: geo.save() except: print('Problem saving: ' + str(man_obj.uuid)) quit()
def reproject_polygon(self, coordinates, geometry_type='Polygon'): """Reprojects coordinates for a GeoJSON polygon""" new_coordinates = [] for ring_list in coordinates: new_ring_list = self.reproject_coordinate_ring(ring_list) new_coordinates.append(new_ring_list) v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates(geometry_type, new_coordinates) if not c_ok: new_coordinates = v_geojson.fix_geometry_rings_dir(geometry_type, new_coordinates) return new_coordinates
def package_coordinates(self, out_x, out_y, geometry_type=None): """Packages coordinates into tuples, lists or list of list objects useful for generating GeoJSON features """ coords = self.make_coordinate_list(out_x, out_y) if len(coords) == 1: return coords[0] if geometry_type == 'Polygon': coords.append(coords[0]) # add the first coordinate to the end to close loop coordinates = [coords] # put the coordinates inside a list to make an outer ring. v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates(geometry_type, coordinates) if not c_ok: coordinates = v_geojson.fix_geometry_rings_dir(geometry_type, coordinates) return coordinates return coords
def make_clusters_geojson(self): """Makes a feature collection of aggregated feature geometries """ geojson = LastUpdatedOrderedDict() geojson['type'] = 'FeatureCollection' geojson['features'] = [] for cluster_group, feat_geoms in self.raw_features.items(): geoms = [] for feat_geometry in feat_geoms: geom_raw = shape(feat_geometry) geom = geom_raw.buffer(self.buffer_tolerance) if geom.is_valid: geoms.append(geom) if len(geoms) < 1: # No valid geometries for this cluster, so skip continue union_geom = unary_union(geoms) poly_union = union_geom.convex_hull poly_union_simple = poly_union.simplify(self.simplify_tolerance) feat = LastUpdatedOrderedDict() feat['type'] = 'Feature' feat['properties'] = LastUpdatedOrderedDict() feat['properties'][self.cluster_property] = cluster_group feat['properties'][self.cluster_num_features_prop] = len(geoms) feat['geometry'] = LastUpdatedOrderedDict() geometry_type = poly_union_simple.geom_type coordinates = [list(poly_union_simple.exterior.coords)] v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates( geometry_type, coordinates) if not c_ok: coordinates = v_geojson.fix_geometry_rings_dir( geometry_type, coordinates) feat['geometry']['type'] = geometry_type feat['geometry']['coordinates'] = coordinates centroid = self.get_feature_centroid(feat['geometry']) feat['properties']['latitude'] = centroid[1] feat['properties']['longitude'] = centroid[0] geojson['features'].append(feat) return geojson
def add_update_geo_data(self, post_data): """ Updates a file associated with a media item """ ok = True errors = [] note = '' required_params = [ 'source_id', 'hash_id', 'meta_type', 'ftype', 'feature_id', 'latitude', 'longitude', 'specificity', 'geojson' ] for r_param in required_params: if r_param not in post_data: # we're missing some required data # don't create the item ok = False message = 'Missing paramater: ' + r_param + '' errors.append(message) note = '; '.join(errors) if ok: coordinates = '' hash_id = post_data['hash_id'].strip() if len(hash_id) < 1: hash_id = False source_id = post_data['source_id'].strip() ftype = post_data['ftype'].strip() # validate the dates output = self.validate_float_param('latitude', post_data, errors) latitude = output['float'] errors = output['errors'] output = self.validate_float_param('longitude', post_data, errors) longitude = output['float'] errors = output['errors'] output = self.validate_int_param('feature_id', post_data, errors) feature_id = output['integer'] errors = output['errors'] output = self.validate_int_param('specificity', post_data, errors) specificity = output['integer'] errors = output['errors'] # now check for GeoJSON, and validate if added coordinates = '' geojson_str = post_data['geojson'].strip() if len(geojson_str) > 1: # we have a geojson object output = self.validate_geojson(geojson_str, errors) coordinates = output['coordinates'] if output['type'] is not False: ftype = output['type'] if isinstance(coordinates, str): v_geojson = ValidateGeoJson() coord_list = json.loads(coordinates) coord_list = v_geojson.fix_geometry_rings_dir( ftype, coord_list) coordinates = json.dumps(coord_list) gg = GeospaceGeneration() lon_lat = gg.get_centroid_lonlat_coordinates( coordinates, ftype) latitude = lon_lat[1] longitude = lon_lat[0] if coordinates is False and latitude is not False\ and longitude is not False: # no coordinates input in GeoJSON so make them ftype = 'Point' coordinates = json.dumps([longitude, latitude], indent=4, ensure_ascii=False) if latitude is False or longitude is False \ or feature_id is False or coordinates is False \ or specificity is False: # 1 or more of the geospatial data are bad ok = False note = '; '.join(errors) else: act_geo = None if hash_id is not False: try: act_geo = Geospace.objects.get(hash_id=hash_id) except Geospace.DoesNotExist: act_geo = False ok = False message = 'Cannot find geo record for hash_id: ' + str( hash_id) errors.append(message) note = '; '.join(errors) if ok: # we're OK to add to an event if act_geo is None: act_geo = Geospace() act_geo.uuid = self.manifest.uuid act_geo.project_uuid = self.manifest.project_uuid act_geo.source_id = source_id act_geo.meta_type = post_data['meta_type'].strip() act_geo.ftype = ftype act_geo.feature_id = feature_id act_geo.latitude = latitude act_geo.longitude = longitude act_geo.coordinates = coordinates act_geo.specificity = specificity act_geo.save() note = 'Updated geodata for ' + self.manifest.uuid if ok: # now clear the cache a change was made cache.clear() else: self.errors = errors self.response = { 'action': 'add-update-geo-data', 'ok': ok, 'change': { 'note': note } } return self.response
def process_solr_polygons(self, solr_polygons): """ processes the solr_json discovery geo tiles, aggregating to a certain depth """ if self.response_zoom_scope >= self.polygon_min_zoom_scope: # we're at a zoom level small enough to make it # worthwile to return complex contained-in polygon features self.get_polygon_db_objects(solr_polygons) i = 0 cnt_i = -1 for poly_key in solr_polygons[::2]: cnt_i += 2 solr_facet_count = solr_polygons[cnt_i] parsed_key = self.parse_solr_value_parts(poly_key) # print('Key: ' + str(parsed_key)) uuid = parsed_key['uuid'] if isinstance(uuid, str): if uuid in self.subjects_objs \ and uuid in self.geo_objs: # we have Subjects and Geospatial models for this # uuid subj_obj = self.subjects_objs[uuid] geo_obj = self.geo_objs[uuid] i += 1 fl = FilterLinks() fl.base_request_json = self.filter_request_dict_json fl.spatial_context = self.spatial_context new_rparams = fl.add_to_request( 'path', subj_obj.context) record = LastUpdatedOrderedDict() record['id'] = fl.make_request_url(new_rparams) record['json'] = fl.make_request_url( new_rparams, '.json') record['count'] = solr_facet_count record['type'] = 'Feature' record['category'] = 'oc-api:geo-contained-in-feature' if self.min_date is not False \ and self.max_date is not False: when = LastUpdatedOrderedDict() when['id'] = '#event-feature-' + uuid when['type'] = 'oc-gen:formation-use-life' # convert numeric to GeoJSON-LD ISO 8601 when['start'] = ISOyears().make_iso_from_float( self.min_date) when['stop'] = ISOyears().make_iso_from_float( self.max_date) record['when'] = when geometry = LastUpdatedOrderedDict() geometry['id'] = '#geo-disc-feature-geom-' + uuid geometry['type'] = geo_obj.ftype coord_obj = json.loads(geo_obj.coordinates) v_geojson = ValidateGeoJson() coord_obj = v_geojson.fix_geometry_rings_dir( geo_obj.ftype, coord_obj) geometry['coordinates'] = coord_obj record['geometry'] = geometry properties = LastUpdatedOrderedDict() properties['id'] = '#geo-disc-feature-' + uuid properties['href'] = record['id'] properties['item-href'] = parsed_key['href'] properties['label'] = subj_obj.context properties['feature-type'] = 'containing-region' properties['count'] = solr_facet_count properties['early bce/ce'] = self.min_date properties['late bce/ce'] = self.max_date record['properties'] = properties dump = json.dumps(record, ensure_ascii=False, indent=4) geojson_obj = geojson.loads(dump) self.geojson_features.append(record)
def make_geo_contained_in_facet_options(self, solr_json): """Gets geospace item query set from a list of options tuples""" geosource_path_keys = (configs.FACETS_SOLR_ROOT_PATH_KEYS + ['disc_geosource']) geosource_val_count_list = utilities.get_dict_path_value( geosource_path_keys, solr_json, default=[]) if not len(geosource_val_count_list): return None # Make the list of tile, count tuples. options_tuples = utilities.get_facet_value_count_tuples( geosource_val_count_list) if not len(options_tuples): return None uuids = [] parsed_solr_entities = {} uuid_geo_dict = {} for solr_entity_str, count in options_tuples: parsed_entity = utilities.parse_solr_encoded_entity_str( solr_entity_str, base_url=self.base_url) if not parsed_entity: logger.warn( 'Cannot parse entity from {}'.format(solr_entity_str)) continue if not '/' in parsed_entity['uri']: logger.warn('Invalid uri from {}'.format(solr_entity_str)) continue uri_parts = parsed_entity['uri'].split('/') uuid = uri_parts[-1] parsed_entity['uuid'] = uuid parsed_solr_entities[solr_entity_str] = parsed_entity uuids.append(uuid) # Make a dictionary of geospace objects keyed by uuid. This # will hit the database in one query to get all geospace # objects not present in the cache. uuid_geo_dict = self._make_cache_geospace_obj_dict(uuids) # Make a dict of context paths, keyed by uuid. This will also # hit the database in only 1 query, for all context paths not # already present in the cache. uuid_context_dict = self._get_cache_contexts_dict(uuids) # Now make the final geo_options = [] for solr_entity_str, count in options_tuples: if solr_entity_str not in parsed_solr_entities: # This solr_entity_str did not validate to extract a UUID. continue parsed_entity = parsed_solr_entities[solr_entity_str] uuid = parsed_entity['uuid'] geo_obj = uuid_geo_dict.get(uuid) if geo_obj is None: logger.warn('No geospace object for {}'.format(uuid)) continue context_path = uuid_context_dict.get(uuid) if context_path is None: logger.warn('No context path for {}'.format(uuid)) continue sl = SearchLinks(request_dict=copy.deepcopy(self.request_dict), base_search_url=self.base_search_url) # Remove non search related params. sl.remove_non_query_params() # Update the request dict for this facet option. sl.replace_param_value( 'path', match_old_value=None, new_value=context_path, ) urls = sl.make_urls_from_request_dict() # NOTE: We're not checking if the URLs are the same # as the current search URL, because part of the point # of listing these features is for visualization display # in the front end. option = LastUpdatedOrderedDict() # The fragment id in the URLs are so we don't have an # ID collision with context facets. option['id'] = urls['html'] + '#geo-in' option['json'] = urls['json'] + '#geo-in' option['count'] = count option['type'] = 'Feature' option['category'] = 'oc-api:geo-contained-in-feature' # Add some general chronology information to the # geospatial feature. option = self._add_when_object_to_feature_option( uuid, option, ) # Add the geometry from the geo_obj coordinates. First # check to make sure they are OK with the the GeoJSON # right-hand rule. geometry = LastUpdatedOrderedDict() geometry['id'] = '#geo-in-geom-{}'.format(uuid) geometry['type'] = geo_obj.ftype coord_obj = json.loads(geo_obj.coordinates) v_geojson = ValidateGeoJson() coord_obj = v_geojson.fix_geometry_rings_dir( geo_obj.ftype, coord_obj) geometry['coordinates'] = coord_obj option['geometry'] = geometry properties = LastUpdatedOrderedDict() properties['id'] = '#geo-in-props-{}'.format(uuid) properties['href'] = option['id'] properties['item-href'] = parsed_entity['uri'] properties['label'] = context_path properties['feature-type'] = 'containing-region' properties['count'] = count properties['early bce/ce'] = self.min_date properties['late bce/ce'] = self.max_date option['properties'] = properties geo_options.append(option) return geo_options
def save_partial_clean_file(self, json_obj, act_dir, filename, id_prop, ok_ids=[], add_props={}, combine_json_obj=None): """ saves a new json file with clean cordinates (to facilitate debugging) """ all_ids = False if not ok_ids: all_ids = True new_json = LastUpdatedOrderedDict() new_json['type'] = 'FeatureCollection' new_json['features'] = [] for feature in json_obj['features']: min_lon = None max_lon = None min_lat = None max_lat = None if all_ids or id_prop in feature['properties']: feature_id = feature['properties'][id_prop] feature['id'] = feature_id if all_ids or feature_id in ok_ids: if feature_id in add_props: id_add_props = add_props[feature_id] for key, value in id_add_props.items(): feature['properties'][key] = value if key == 'uri': uuid = value.split('/')[-1] sub = Subject.objects.get(uuid=uuid) feature['properties'][ 'context'] = sub.context.replace( 'Italy/', '') asses = Assertion.objects.filter( uuid=uuid, object_type='documents') d_uuids = [] for ass in asses: if ass.object_uuid not in d_uuids: d_uuids.append(ass.object_uuid) d_mans = Manifest.objects.filter( uuid__in=d_uuids) min_len = 10000000 for d_man in d_mans: if len(d_man.label) < min_len: min_len = len(d_man.label) feature['properties'][ 'trench-book'] = d_man.label geometry_type = feature['geometry']['type'] coordinates = feature['geometry']['coordinates'] v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates( geometry_type, coordinates) if not c_ok: coordinates = v_geojson.fix_geometry_rings_dir( geometry_type, coordinates) feature['geometry']['coordinates'] = coordinates if geometry_type == 'Polygon': poly = Polygon(coordinates) act_feature = geojson.Feature(geometry=poly) cors = geojson.utils.coords(act_feature) for cor in cors: if min_lon is None or min_lon > cor[0]: min_lon = cor[0] if max_lon is None or max_lon < cor[0]: max_lon = cor[0] if min_lat is None or min_lat > cor[1]: min_lat = cor[1] if max_lat is None or max_lat < cor[1]: max_lat = cor[1] if combine_json_obj: feature['properties']['p-uris'] = '' print('Limit to {}, {} :: {}, {}'.format( min_lon, min_lat, max_lon, max_lat)) near_contexts = [] near_uris = [] contexts = [] uris = [] for cfeature in combine_json_obj['features']: near = True inside = False cgeometry_type = cfeature['geometry']['type'] if cgeometry_type == 'Point': ccors = cfeature['geometry']['coordinates'] if ccors[0] < min_lon or ccors[0] > max_lon: near = False if ccors[1] < min_lat or ccors[1] > max_lat: near = False spoly = shape(feature['geometry']) point = Point(ccors) # create point inside = spoly.contains(point) # print('inside?: {}'.format(inside)) if 'uri' in cfeature['properties'] and ( near or inside): uri = cfeature['properties']['uri'] if inside: uris.append(uri) if near: near_uris.append(uri) uuid = uri.split('/')[-1] sub = Subject.objects.get(uuid=uuid) context = '/'.join( sub.context.split('/')[0:5]) if near: near_contexts.append(context) if inside: contexts.append(context) # new_json['features'].append(cfeature) n_common_context, n_all_contexts, n_c_uuid = self.make_context_count_str( near_contexts) common_context, all_contexts, c_uuid = self.make_context_count_str( contexts) feature['properties']['p-uris'] = '; '.join(uris) feature['properties'][ 'n-contexts'] = n_all_contexts feature['properties'][ 'n-context'] = n_common_context feature['properties']['n-c-uuid'] = n_c_uuid feature['properties']['contexts'] = all_contexts feature['properties']['context'] = common_context feature['properties']['c-uuid'] = c_uuid new_json['features'].append(feature) dir_file = self.set_check_directory( act_dir) + '/id-clean-coord-' + filename self.save_json_file(new_json, None, None, dir_file=dir_file)