def add_sites_geo_from_filecache(self): """ updates Open Context to save new sites location information """ if self.filecache is not None: # print('Cache update !: ' + self.cache_filekey) self.filecache.working_dir = self.working_dir json_obj = self.filecache.get_dict_from_file(self.cache_filekey) if isinstance(json_obj, dict): if 'trinomial_refs' in json_obj: for tri_ref in json_obj['trinomial_refs']: for trinomial in tri_ref['trinomials']: man_objs = Manifest.objects.filter(label=trinomial, class_uri='oc-gen:cat-site') if len(man_objs) > 0: man_obj = man_objs[0] county_code = trinomial[0:4] if county_code in self.COUNTY_GEO: county_geo = self.COUNTY_GEO[county_code] print('County code: ' + county_code + ' is ' + str(county_geo)) geos = Geospace.objects.filter(uuid=man_obj.uuid)[:1] if len(geos) < 1: geo = Geospace() geo.uuid = man_obj.uuid geo.project_uuid = man_obj.project_uuid geo.source_id = self.source_id geo.item_type = man_obj.item_type geo.feature_id = 1 geo.meta_type = "oc-gen:discovey-location" geo.ftype = 'Point' geo.latitude = county_geo['lat'] geo.longitude = county_geo['lon'] geo.specificity = -11 geo.save()
def create_geospatial_data_from_geonames_json(self, manifest, json_data): """ creates and saves geospatial data derived from geonames """ ok = False coords = '' lat = False lon = False ftype = 'Point' if 'bbox' in json_data: ftype = 'Polygon' east = json_data['bbox']['east'] south = json_data['bbox']['south'] west = json_data['bbox']['west'] north = json_data['bbox']['north'] lat = (south + north) / 2 lon = (east + west) / 2 coordinates = [] cood_inner = [] cood_inner.append([east, south]) cood_inner.append([east, north]) cood_inner.append([west, north]) cood_inner.append([west, south]) cood_inner.append([east, south]) coordinates.append(cood_inner) coords = json.dumps(coordinates, indent=4, ensure_ascii=False) else: coords = '' if 'lat' in json_data: lat = float(json_data['lat']) if 'lng' in json_data: lon = float(json_data['lng']) if lat is not False and lon is not False: # we have OK coodinates, let's save them geodata = Geospace() geodata.uuid = manifest.uuid geodata.project_uuid = manifest.project_uuid geodata.source_id = 'geonames-api' geodata.item_type = manifest.item_type geodata.feature_id = 1 geodata.meta_type = 'oc-gen:discovey-location' geodata.ftype = ftype geodata.latitude = lat geodata.longitude = lon geodata.specificity = self.specifity geodata.coordinates = coords geodata.note = 'Location data from GeoNames.org' geodata.save() ok = True if ok: self.new_geodata += 1 output = 'New geodata [' + str(self.new_geodata) + '] ' + str(manifest.label) output += ' (' + str(manifest.uuid) + ')' print(output) return ok
def add_feature_to_existing_items(self, feature): """ Process a feature to extract geospatial object. It will: (1) Find the appropriate item in the manifest table (2) Adds a record in the geospace table """ man_obj = None if 'properties' in feature: props = feature['properties'] if 'uri' in props: try_uuid = props['uri'].split('/')[-1] man_objs = Manifest.objects.filter(uuid=try_uuid)[:1] if man_objs: man_obj = man_objs[0] if man_obj and 'geometry' in feature: # first get and validate the coordinates from the GeoJSON file if man_obj.uuid not in self.uuid_counts: self.uuid_counts[man_obj.uuid] = 0 self.uuid_counts[man_obj.uuid] += 1 geometry_type = feature['geometry']['type'] coordinates = feature['geometry']['coordinates'] v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates(geometry_type, coordinates) if not c_ok: print('Fixing coordinates for: {}'.format(man_obj.uuid)) coordinates = v_geojson.fix_geometry_rings_dir(geometry_type, coordinates) if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2: Geospace.objects.filter(uuid=man_obj.uuid).delete() coord_str = json.dumps(coordinates, indent=4, ensure_ascii=False) gg = GeospaceGeneration() lon_lat = gg.get_centroid_lonlat_coordinates(coord_str, geometry_type) print('Saving new geomettry for: ' + str(man_obj.uuid)) geo = Geospace() geo.uuid = man_obj.uuid geo.project_uuid = man_obj.project_uuid geo.source_id = self.source_id geo.item_type = man_obj.item_type geo.feature_id = self.uuid_counts[man_obj.uuid] geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION geo.ftype = geometry_type geo.latitude = lon_lat[1] geo.longitude = lon_lat[0] geo.specificity = 0 # dump coordinates as json string geo.coordinates = coord_str try: geo.save() except: print('Problem saving: ' + str(man_obj.uuid)) quit()
def create_geospatial_data_from_geonames_json(self, manifest, json_data): """ creates and saves geospatial data derived from geonames """ ok = False coords = '' lat = False lon = False ftype = 'Point' if 'bbox' in json_data: ftype = 'Polygon' east = json_data['bbox']['east'] south = json_data['bbox']['south'] west = json_data['bbox']['west'] north = json_data['bbox']['north'] lat = (south + north) / 2 lon = (east + west) / 2 coordinates = [] cood_inner = [] cood_inner.append([east, south]) cood_inner.append([east, north]) cood_inner.append([west, north]) cood_inner.append([west, south]) cood_inner.append([east, south]) coordinates.append(cood_inner) coords = json.dumps(coordinates, indent=4, ensure_ascii=False) else: coords = '' if 'lat' in json_data: lat = float(json_data['lat']) if 'lng' in json_data: lon = float(json_data['lng']) if lat is not False and lon is not False: # we have OK coodinates, let's save them geodata = Geospace() geodata.uuid = manifest.uuid geodata.project_uuid = manifest.project_uuid geodata.source_id = 'geonames-api' geodata.item_type = manifest.item_type geodata.feature_id = 1 geodata.meta_type = 'oc-gen:discovey-location' geodata.ftype = ftype geodata.latitude = lat geodata.longitude = lon geodata.specificity = self.specifity geodata.coordinates = coords geodata.note = 'Location data from GeoNames.org' geodata.save() ok = True if ok: self.new_geodata += 1 output = 'New geodata [' + str(self.new_geodata) + '] ' + str( manifest.label) output += ' (' + str(manifest.uuid) + ')' print(output) return ok
def add_feature_to_existing_items(self, feature): """ Process a feature to extract geospatial object. It will: (1) Find the appropriate item in the manifest table (2) Adds a record in the geospace table """ man_obj = None if 'properties' in feature: props = feature['properties'] if 'uri' in props: try_uuid = props['uri'].split('/')[-1] man_objs = Manifest.objects.filter(uuid=try_uuid)[:1] if man_objs: man_obj = man_objs[0] if man_obj and 'geometry' in feature: # first get and validate the coordinates from the GeoJSON file if man_obj.uuid not in self.uuid_counts: self.uuid_counts[man_obj.uuid] = 0 self.uuid_counts[man_obj.uuid] += 1 geometry_type = feature['geometry']['type'] coordinates = feature['geometry']['coordinates'] v_geojson = ValidateGeoJson() c_ok = v_geojson.validate_all_geometry_coordinates( geometry_type, coordinates) if not c_ok: print('Fixing coordinates for: {}'.format(man_obj.uuid)) coordinates = v_geojson.fix_geometry_rings_dir( geometry_type, coordinates) if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2: Geospace.objects.filter(uuid=man_obj.uuid).delete() coord_str = json.dumps(coordinates, indent=4, ensure_ascii=False) gg = GeospaceGeneration() lon_lat = gg.get_centroid_lonlat_coordinates( coord_str, geometry_type) print('Saving new geomettry for: ' + str(man_obj.uuid)) geo = Geospace() geo.uuid = man_obj.uuid geo.project_uuid = man_obj.project_uuid geo.source_id = self.source_id geo.item_type = man_obj.item_type geo.feature_id = self.uuid_counts[man_obj.uuid] geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION geo.ftype = geometry_type geo.latitude = lon_lat[1] geo.longitude = lon_lat[0] geo.specificity = 0 # dump coordinates as json string geo.coordinates = coord_str try: geo.save() except: print('Problem saving: ' + str(man_obj.uuid)) quit()
def make_geo_obj(self, feature_id, lon, lat, coords=False): geo_obj = Geospace() geo_obj.uuid = self.project_uuid geo_obj.project_uuid = self.project_uuid geo_obj.source_id = 'Project metadata summary' geo_obj.item_type = 'projects' geo_obj.feature_id = feature_id geo_obj.meta_type = 'oc-gen:geo-coverage' if coords is False: geo_obj.ftype = 'Point' geo_obj.coordinates = '' else: geo_obj.ftype = 'Polygon' geo_obj.coordinates = json.dumps(coords, ensure_ascii=False) geo_obj.latitude = lat geo_obj.longitude = lon geo_obj.specificity = 0 geo_obj.note = 'Project geographic coverage \ summarized from geospatial data \ describing subjects published \ with this project.' return geo_obj
def make_geo_obj(self, feature_id, lon, lat, coords=False): geo_obj = Geospace() geo_obj.uuid = self.project_uuid geo_obj.project_uuid = self.project_uuid geo_obj.source_id = 'Project metadata summary' geo_obj.item_type = 'projects' geo_obj.feature_id = feature_id geo_obj.meta_type = 'oc-gen:geo-coverage' if coords is False: geo_obj.ftype = 'Point' geo_obj.coordinates = '' else: geo_obj.ftype = 'Polygon' geo_obj.coordinates = json.dumps(coords, ensure_ascii=False) geo_obj.latitude = lat geo_obj.longitude = lon geo_obj.specificity = self.project_specificity geo_obj.note = 'Project geographic coverage \ summarized from geospatial data \ describing subjects published \ with this project.' return geo_obj
def add_sites_geo_from_filecache(self): """ updates Open Context to save new sites location information """ if self.filecache is not None: # print('Cache update !: ' + self.cache_filekey) self.filecache.working_dir = self.working_dir json_obj = self.filecache.get_dict_from_file(self.cache_filekey) if isinstance(json_obj, dict): if 'trinomial_refs' in json_obj: for tri_ref in json_obj['trinomial_refs']: for trinomial in tri_ref['trinomials']: man_objs = Manifest.objects.filter( label=trinomial, class_uri='oc-gen:cat-site') if len(man_objs) > 0: man_obj = man_objs[0] county_code = trinomial[0:4] if county_code in self.COUNTY_GEO: county_geo = self.COUNTY_GEO[county_code] print('County code: ' + county_code + ' is ' + str(county_geo)) geos = Geospace.objects.filter( uuid=man_obj.uuid)[:1] if len(geos) < 1: geo = Geospace() geo.uuid = man_obj.uuid geo.project_uuid = man_obj.project_uuid geo.source_id = self.source_id geo.item_type = man_obj.item_type geo.feature_id = 1 geo.meta_type = "oc-gen:discovey-location" geo.ftype = 'Point' geo.latitude = county_geo['lat'] geo.longitude = county_geo['lon'] geo.specificity = -11 geo.save()
def process_geojson_item(self, subject_field_num, subject_in_rows, subject_uuid): """ adds geojson data if it exists for an item """ if subject_field_num in self.geojson_rels: geojson_field_num = self.geojson_rels[subject_field_num] geojson_recs = ImportCell.objects\ .filter(source_id=self.source_id, field_num=geojson_field_num, row_num__in=subject_in_rows)\ .exclude(record='') # the rows object now has a list of all the rows, with validated coordinate # data. Now we can add these to the database! geo_keys_done = [] gg = GeospaceGeneration() for geojson_rec in geojson_recs: geo_key = geojson_rec.rec_hash record = geojson_rec.record if geo_key not in geo_keys_done: geo_keys_done.append(geo_key) try: json_obj = json.loads(record) except: json_obj = False if isinstance(json_obj, dict): if 'type' in json_obj \ and 'coordinates' in json_obj: # OK we've got good data lon_lat = gg.get_centroid_lonlat_coordinates(record, json_obj['type']) if isinstance(lon_lat, tuple): # print('Good centroid for ' + str(subject_uuid)) # great! We have centroid lon_lat # this is another data validation passed coordinates = json_obj['coordinates'] same_geos = Geospace.objects\ .filter(uuid=subject_uuid, coordinates=coordinates)[:1] if len(same_geos) < 1: # OK it is valid and does not exist for the item # we can add it geo = Geospace() geo.uuid = str(subject_uuid) geo.project_uuid = self.project_uuid geo.source_id = self.source_id geo.item_type = 'subjects' geo.feature_id = len(geo_keys_done) geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION geo.ftype = json_obj['type'] geo.latitude = lon_lat[1] geo.longitude = lon_lat[0] geo.specificity = 0 # dump coordinates as json string geo.coordinates = json.dumps(coordinates, indent=4, ensure_ascii=False) try: geo.save() except: print('Did not like ' + str(row) + ' with ' + str(subject_uuid)) quit()
def process_geospace_item(self, subject_field_num, subject_in_rows, subject_uuid): """ adds lat lon data if it exists for an item """ if subject_field_num in self.geospace_fields: # this subject field has associated lat - lon data # now make a list where of rows that have non-blank lat data act_geo_fields = self.geospace_fields[subject_field_num] rows = {} lat_recs = ImportCell.objects\ .filter(source_id=self.source_id, field_num=act_geo_fields['lat'], row_num__in=subject_in_rows)\ .exclude(record='') for lat_rec in lat_recs: rows[lat_rec.row_num] = {'lat': False, 'lon': False} rows[lat_rec.row_num]['lat'] = self.validate_geo_coordinate(lat_rec.record, 'lat') lon_recs = ImportCell.objects\ .filter(source_id=self.source_id, field_num=act_geo_fields['lon'], row_num__in=subject_in_rows)\ .exclude(record='') for lon_rec in lon_recs: if lon_rec.row_num not in rows: rows[lon_rec.row_num] = {'lat': False, 'lon': False} rows[lon_rec.row_num]['lon'] = self.validate_geo_coordinate(lon_rec.record, 'lon') # the rows object now has a list of all the rows, with validated coordinate # data. Now we can add these to the database! geo_feature = 1 geo_keys_done = [] for row_key, row in rows.items(): if row['lat'] is not False and \ row['lon'] is not False: geo_key = str(row['lat']) + ',' + str(row['lon']) if geo_key not in geo_keys_done: # we havent checked this set of coordinates yet geo_keys_done.append(geo_key) # now check to make sure we don't already have these coordinates # on this item same_geos = Geospace.objects\ .filter(uuid=subject_uuid, latitude=row['lat'], longitude=row['lon'])[:1] if len(same_geos) < 1: # it is a new coordinate, ok to add geo = Geospace() geo.uuid = str(subject_uuid) geo.project_uuid = self.project_uuid geo.source_id = self.source_id geo.item_type = 'subjects' geo.feature_id = geo_feature geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION geo.ftype = 'Point' geo.latitude = row['lat'] geo.longitude = row['lon'] geo.specificity = 0 # dump coordinates as json string in lon - lat (GeoJSON order) geo.coordinates = json.dumps([row['lon'], row['lat']], indent=4, ensure_ascii=False) try: geo.save() geo_feature += 1 except: print('Did not like ' + str(row) + ' with ' + str(subject_uuid)) quit()