def add_sites_geo_from_filecache(self):
""" updates Open Context to save new sites
location information
"""
if self.filecache is not None:
# print('Cache update !: ' + self.cache_filekey)
self.filecache.working_dir = self.working_dir
json_obj = self.filecache.get_dict_from_file(self.cache_filekey)
if isinstance(json_obj, dict):
if 'trinomial_refs' in json_obj:
for tri_ref in json_obj['trinomial_refs']:
for trinomial in tri_ref['trinomials']:
man_objs = Manifest.objects.filter(label=trinomial,
class_uri='oc-gen:cat-site')
if len(man_objs) > 0:
man_obj = man_objs[0]
county_code = trinomial[0:4]
if county_code in self.COUNTY_GEO:
county_geo = self.COUNTY_GEO[county_code]
print('County code: ' + county_code + ' is ' + str(county_geo))
geos = Geospace.objects.filter(uuid=man_obj.uuid)[:1]
if len(geos) < 1:
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = 1
geo.meta_type = "oc-gen:discovey-location"
geo.ftype = 'Point'
geo.latitude = county_geo['lat']
geo.longitude = county_geo['lon']
geo.specificity = -11
geo.save()
def create_geospatial_data_from_geonames_json(self, manifest, json_data):
""" creates and saves geospatial data derived from geonames """
ok = False
coords = ''
lat = False
lon = False
ftype = 'Point'
if 'bbox' in json_data:
ftype = 'Polygon'
east = json_data['bbox']['east']
south = json_data['bbox']['south']
west = json_data['bbox']['west']
north = json_data['bbox']['north']
lat = (south + north) / 2
lon = (east + west) / 2
coordinates = []
cood_inner = []
cood_inner.append([east,
south])
cood_inner.append([east,
north])
cood_inner.append([west,
north])
cood_inner.append([west,
south])
cood_inner.append([east,
south])
coordinates.append(cood_inner)
coords = json.dumps(coordinates,
indent=4,
ensure_ascii=False)
else:
coords = ''
if 'lat' in json_data:
lat = float(json_data['lat'])
if 'lng' in json_data:
lon = float(json_data['lng'])
if lat is not False and lon is not False:
# we have OK coodinates, let's save them
geodata = Geospace()
geodata.uuid = manifest.uuid
geodata.project_uuid = manifest.project_uuid
geodata.source_id = 'geonames-api'
geodata.item_type = manifest.item_type
geodata.feature_id = 1
geodata.meta_type = 'oc-gen:discovey-location'
geodata.ftype = ftype
geodata.latitude = lat
geodata.longitude = lon
geodata.specificity = self.specifity
geodata.coordinates = coords
geodata.note = 'Location data from GeoNames.org'
geodata.save()
ok = True
if ok:
self.new_geodata += 1
output = 'New geodata [' + str(self.new_geodata) + '] ' + str(manifest.label)
output += ' (' + str(manifest.uuid) + ')'
print(output)
return ok
def add_feature_to_existing_items(self, feature):
""" Process a feature to extract geospatial
object. It will:
(1) Find the appropriate item in the manifest table
(2) Adds a record in the geospace table
"""
man_obj = None
if 'properties' in feature:
props = feature['properties']
if 'uri' in props:
try_uuid = props['uri'].split('/')[-1]
man_objs = Manifest.objects.filter(uuid=try_uuid)[:1]
if man_objs:
man_obj = man_objs[0]
if man_obj and 'geometry' in feature:
# first get and validate the coordinates from the GeoJSON file
if man_obj.uuid not in self.uuid_counts:
self.uuid_counts[man_obj.uuid] = 0
self.uuid_counts[man_obj.uuid] += 1
geometry_type = feature['geometry']['type']
coordinates = feature['geometry']['coordinates']
v_geojson = ValidateGeoJson()
c_ok = v_geojson.validate_all_geometry_coordinates(geometry_type,
coordinates)
if not c_ok:
print('Fixing coordinates for: {}'.format(man_obj.uuid))
coordinates = v_geojson.fix_geometry_rings_dir(geometry_type,
coordinates)
if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2:
Geospace.objects.filter(uuid=man_obj.uuid).delete()
coord_str = json.dumps(coordinates,
indent=4,
ensure_ascii=False)
gg = GeospaceGeneration()
lon_lat = gg.get_centroid_lonlat_coordinates(coord_str, geometry_type)
print('Saving new geomettry for: ' + str(man_obj.uuid))
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = self.uuid_counts[man_obj.uuid]
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = geometry_type
geo.latitude = lon_lat[1]
geo.longitude = lon_lat[0]
geo.specificity = 0
# dump coordinates as json string
geo.coordinates = coord_str
try:
geo.save()
except:
print('Problem saving: ' + str(man_obj.uuid))
quit()
def create_geospatial_data_from_geonames_json(self, manifest, json_data):
""" creates and saves geospatial data derived from geonames """
ok = False
coords = ''
lat = False
lon = False
ftype = 'Point'
if 'bbox' in json_data:
ftype = 'Polygon'
east = json_data['bbox']['east']
south = json_data['bbox']['south']
west = json_data['bbox']['west']
north = json_data['bbox']['north']
lat = (south + north) / 2
lon = (east + west) / 2
coordinates = []
cood_inner = []
cood_inner.append([east, south])
cood_inner.append([east, north])
cood_inner.append([west, north])
cood_inner.append([west, south])
cood_inner.append([east, south])
coordinates.append(cood_inner)
coords = json.dumps(coordinates, indent=4, ensure_ascii=False)
else:
coords = ''
if 'lat' in json_data:
lat = float(json_data['lat'])
if 'lng' in json_data:
lon = float(json_data['lng'])
if lat is not False and lon is not False:
# we have OK coodinates, let's save them
geodata = Geospace()
geodata.uuid = manifest.uuid
geodata.project_uuid = manifest.project_uuid
geodata.source_id = 'geonames-api'
geodata.item_type = manifest.item_type
geodata.feature_id = 1
geodata.meta_type = 'oc-gen:discovey-location'
geodata.ftype = ftype
geodata.latitude = lat
geodata.longitude = lon
geodata.specificity = self.specifity
geodata.coordinates = coords
geodata.note = 'Location data from GeoNames.org'
geodata.save()
ok = True
if ok:
self.new_geodata += 1
output = 'New geodata [' + str(self.new_geodata) + '] ' + str(
manifest.label)
output += ' (' + str(manifest.uuid) + ')'
print(output)
return ok
def add_feature_to_existing_items(self, feature):
""" Process a feature to extract geospatial
object. It will:
(1) Find the appropriate item in the manifest table
(2) Adds a record in the geospace table
"""
man_obj = None
if 'properties' in feature:
props = feature['properties']
if 'uri' in props:
try_uuid = props['uri'].split('/')[-1]
man_objs = Manifest.objects.filter(uuid=try_uuid)[:1]
if man_objs:
man_obj = man_objs[0]
if man_obj and 'geometry' in feature:
# first get and validate the coordinates from the GeoJSON file
if man_obj.uuid not in self.uuid_counts:
self.uuid_counts[man_obj.uuid] = 0
self.uuid_counts[man_obj.uuid] += 1
geometry_type = feature['geometry']['type']
coordinates = feature['geometry']['coordinates']
v_geojson = ValidateGeoJson()
c_ok = v_geojson.validate_all_geometry_coordinates(
geometry_type, coordinates)
if not c_ok:
print('Fixing coordinates for: {}'.format(man_obj.uuid))
coordinates = v_geojson.fix_geometry_rings_dir(
geometry_type, coordinates)
if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2:
Geospace.objects.filter(uuid=man_obj.uuid).delete()
coord_str = json.dumps(coordinates, indent=4, ensure_ascii=False)
gg = GeospaceGeneration()
lon_lat = gg.get_centroid_lonlat_coordinates(
coord_str, geometry_type)
print('Saving new geomettry for: ' + str(man_obj.uuid))
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = self.uuid_counts[man_obj.uuid]
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = geometry_type
geo.latitude = lon_lat[1]
geo.longitude = lon_lat[0]
geo.specificity = 0
# dump coordinates as json string
geo.coordinates = coord_str
try:
geo.save()
except:
print('Problem saving: ' + str(man_obj.uuid))
quit()
def make_geo_obj(self, feature_id, lon, lat, coords=False):
geo_obj = Geospace()
geo_obj.uuid = self.project_uuid
geo_obj.project_uuid = self.project_uuid
geo_obj.source_id = 'Project metadata summary'
geo_obj.item_type = 'projects'
geo_obj.feature_id = feature_id
geo_obj.meta_type = 'oc-gen:geo-coverage'
if coords is False:
geo_obj.ftype = 'Point'
geo_obj.coordinates = ''
else:
geo_obj.ftype = 'Polygon'
geo_obj.coordinates = json.dumps(coords, ensure_ascii=False)
geo_obj.latitude = lat
geo_obj.longitude = lon
geo_obj.specificity = 0
geo_obj.note = 'Project geographic coverage \
summarized from geospatial data \
describing subjects published \
with this project.'
return geo_obj
def make_geo_obj(self, feature_id, lon, lat, coords=False):
geo_obj = Geospace()
geo_obj.uuid = self.project_uuid
geo_obj.project_uuid = self.project_uuid
geo_obj.source_id = 'Project metadata summary'
geo_obj.item_type = 'projects'
geo_obj.feature_id = feature_id
geo_obj.meta_type = 'oc-gen:geo-coverage'
if coords is False:
geo_obj.ftype = 'Point'
geo_obj.coordinates = ''
else:
geo_obj.ftype = 'Polygon'
geo_obj.coordinates = json.dumps(coords, ensure_ascii=False)
geo_obj.latitude = lat
geo_obj.longitude = lon
geo_obj.specificity = self.project_specificity
geo_obj.note = 'Project geographic coverage \
summarized from geospatial data \
describing subjects published \
with this project.'
return geo_obj
def add_sites_geo_from_filecache(self):
""" updates Open Context to save new sites
location information
"""
if self.filecache is not None:
# print('Cache update !: ' + self.cache_filekey)
self.filecache.working_dir = self.working_dir
json_obj = self.filecache.get_dict_from_file(self.cache_filekey)
if isinstance(json_obj, dict):
if 'trinomial_refs' in json_obj:
for tri_ref in json_obj['trinomial_refs']:
for trinomial in tri_ref['trinomials']:
man_objs = Manifest.objects.filter(
label=trinomial, class_uri='oc-gen:cat-site')
if len(man_objs) > 0:
man_obj = man_objs[0]
county_code = trinomial[0:4]
if county_code in self.COUNTY_GEO:
county_geo = self.COUNTY_GEO[county_code]
print('County code: ' + county_code +
' is ' + str(county_geo))
geos = Geospace.objects.filter(
uuid=man_obj.uuid)[:1]
if len(geos) < 1:
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = 1
geo.meta_type = "oc-gen:discovey-location"
geo.ftype = 'Point'
geo.latitude = county_geo['lat']
geo.longitude = county_geo['lon']
geo.specificity = -11
geo.save()
def process_geojson_item(self,
subject_field_num,
subject_in_rows,
subject_uuid):
""" adds geojson data if it exists for an item
"""
if subject_field_num in self.geojson_rels:
geojson_field_num = self.geojson_rels[subject_field_num]
geojson_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=geojson_field_num,
row_num__in=subject_in_rows)\
.exclude(record='')
# the rows object now has a list of all the rows, with validated coordinate
# data. Now we can add these to the database!
geo_keys_done = []
gg = GeospaceGeneration()
for geojson_rec in geojson_recs:
geo_key = geojson_rec.rec_hash
record = geojson_rec.record
if geo_key not in geo_keys_done:
geo_keys_done.append(geo_key)
try:
json_obj = json.loads(record)
except:
json_obj = False
if isinstance(json_obj, dict):
if 'type' in json_obj \
and 'coordinates' in json_obj:
# OK we've got good data
lon_lat = gg.get_centroid_lonlat_coordinates(record,
json_obj['type'])
if isinstance(lon_lat, tuple):
# print('Good centroid for ' + str(subject_uuid))
# great! We have centroid lon_lat
# this is another data validation passed
coordinates = json_obj['coordinates']
same_geos = Geospace.objects\
.filter(uuid=subject_uuid,
coordinates=coordinates)[:1]
if len(same_geos) < 1:
# OK it is valid and does not exist for the item
# we can add it
geo = Geospace()
geo.uuid = str(subject_uuid)
geo.project_uuid = self.project_uuid
geo.source_id = self.source_id
geo.item_type = 'subjects'
geo.feature_id = len(geo_keys_done)
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = json_obj['type']
geo.latitude = lon_lat[1]
geo.longitude = lon_lat[0]
geo.specificity = 0
# dump coordinates as json string
geo.coordinates = json.dumps(coordinates,
indent=4,
ensure_ascii=False)
try:
geo.save()
except:
print('Did not like ' + str(row) + ' with ' + str(subject_uuid))
quit()
def process_geospace_item(self,
subject_field_num,
subject_in_rows,
subject_uuid):
""" adds lat lon data if it exists for an item
"""
if subject_field_num in self.geospace_fields:
# this subject field has associated lat - lon data
# now make a list where of rows that have non-blank lat data
act_geo_fields = self.geospace_fields[subject_field_num]
rows = {}
lat_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=act_geo_fields['lat'],
row_num__in=subject_in_rows)\
.exclude(record='')
for lat_rec in lat_recs:
rows[lat_rec.row_num] = {'lat': False,
'lon': False}
rows[lat_rec.row_num]['lat'] = self.validate_geo_coordinate(lat_rec.record,
'lat')
lon_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=act_geo_fields['lon'],
row_num__in=subject_in_rows)\
.exclude(record='')
for lon_rec in lon_recs:
if lon_rec.row_num not in rows:
rows[lon_rec.row_num] = {'lat': False,
'lon': False}
rows[lon_rec.row_num]['lon'] = self.validate_geo_coordinate(lon_rec.record,
'lon')
# the rows object now has a list of all the rows, with validated coordinate
# data. Now we can add these to the database!
geo_feature = 1
geo_keys_done = []
for row_key, row in rows.items():
if row['lat'] is not False and \
row['lon'] is not False:
geo_key = str(row['lat']) + ',' + str(row['lon'])
if geo_key not in geo_keys_done:
# we havent checked this set of coordinates yet
geo_keys_done.append(geo_key)
# now check to make sure we don't already have these coordinates
# on this item
same_geos = Geospace.objects\
.filter(uuid=subject_uuid,
latitude=row['lat'],
longitude=row['lon'])[:1]
if len(same_geos) < 1:
# it is a new coordinate, ok to add
geo = Geospace()
geo.uuid = str(subject_uuid)
geo.project_uuid = self.project_uuid
geo.source_id = self.source_id
geo.item_type = 'subjects'
geo.feature_id = geo_feature
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = 'Point'
geo.latitude = row['lat']
geo.longitude = row['lon']
geo.specificity = 0
# dump coordinates as json string in lon - lat (GeoJSON order)
geo.coordinates = json.dumps([row['lon'], row['lat']],
indent=4,
ensure_ascii=False)
try:
geo.save()
geo_feature += 1
except:
print('Did not like ' + str(row) + ' with ' + str(subject_uuid))
quit()
