def create_geospatial_data_from_geonames_json(self, manifest, json_data):
""" creates and saves geospatial data derived from geonames """
ok = False
coords = ''
lat = False
lon = False
ftype = 'Point'
if 'bbox' in json_data:
ftype = 'Polygon'
east = json_data['bbox']['east']
south = json_data['bbox']['south']
west = json_data['bbox']['west']
north = json_data['bbox']['north']
lat = (south + north) / 2
lon = (east + west) / 2
coordinates = []
cood_inner = []
cood_inner.append([east,
south])
cood_inner.append([east,
north])
cood_inner.append([west,
north])
cood_inner.append([west,
south])
cood_inner.append([east,
south])
coordinates.append(cood_inner)
coords = json.dumps(coordinates,
indent=4,
ensure_ascii=False)
else:
coords = ''
if 'lat' in json_data:
lat = float(json_data['lat'])
if 'lng' in json_data:
lon = float(json_data['lng'])
if lat is not False and lon is not False:
# we have OK coodinates, let's save them
geodata = Geospace()
geodata.uuid = manifest.uuid
geodata.project_uuid = manifest.project_uuid
geodata.source_id = 'geonames-api'
geodata.item_type = manifest.item_type
geodata.feature_id = 1
geodata.meta_type = 'oc-gen:discovey-location'
geodata.ftype = ftype
geodata.latitude = lat
geodata.longitude = lon
geodata.specificity = self.specifity
geodata.coordinates = coords
geodata.note = 'Location data from GeoNames.org'
geodata.save()
ok = True
if ok:
self.new_geodata += 1
output = 'New geodata [' + str(self.new_geodata) + '] ' + str(manifest.label)
output += ' (' + str(manifest.uuid) + ')'
print(output)
return ok
def add_sites_geo_from_filecache(self):
""" updates Open Context to save new sites
location information
"""
if self.filecache is not None:
# print('Cache update !: ' + self.cache_filekey)
self.filecache.working_dir = self.working_dir
json_obj = self.filecache.get_dict_from_file(self.cache_filekey)
if isinstance(json_obj, dict):
if 'trinomial_refs' in json_obj:
for tri_ref in json_obj['trinomial_refs']:
for trinomial in tri_ref['trinomials']:
man_objs = Manifest.objects.filter(label=trinomial,
class_uri='oc-gen:cat-site')
if len(man_objs) > 0:
man_obj = man_objs[0]
county_code = trinomial[0:4]
if county_code in self.COUNTY_GEO:
county_geo = self.COUNTY_GEO[county_code]
print('County code: ' + county_code + ' is ' + str(county_geo))
geos = Geospace.objects.filter(uuid=man_obj.uuid)[:1]
if len(geos) < 1:
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = 1
geo.meta_type = "oc-gen:discovey-location"
geo.ftype = 'Point'
geo.latitude = county_geo['lat']
geo.longitude = county_geo['lon']
geo.specificity = -11
geo.save()
def add_feature_to_existing_items(self, feature):
""" Process a feature to extract geospatial
object. It will:
(1) Find the appropriate item in the manifest table
(2) Adds a record in the geospace table
"""
man_obj = None
if 'properties' in feature:
props = feature['properties']
if 'uri' in props:
try_uuid = props['uri'].split('/')[-1]
man_objs = Manifest.objects.filter(uuid=try_uuid)[:1]
if man_objs:
man_obj = man_objs[0]
if man_obj and 'geometry' in feature:
# first get and validate the coordinates from the GeoJSON file
if man_obj.uuid not in self.uuid_counts:
self.uuid_counts[man_obj.uuid] = 0
self.uuid_counts[man_obj.uuid] += 1
geometry_type = feature['geometry']['type']
coordinates = feature['geometry']['coordinates']
v_geojson = ValidateGeoJson()
c_ok = v_geojson.validate_all_geometry_coordinates(geometry_type,
coordinates)
if not c_ok:
print('Fixing coordinates for: {}'.format(man_obj.uuid))
coordinates = v_geojson.fix_geometry_rings_dir(geometry_type,
coordinates)
if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2:
Geospace.objects.filter(uuid=man_obj.uuid).delete()
coord_str = json.dumps(coordinates,
indent=4,
ensure_ascii=False)
gg = GeospaceGeneration()
lon_lat = gg.get_centroid_lonlat_coordinates(coord_str, geometry_type)
print('Saving new geomettry for: ' + str(man_obj.uuid))
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = self.uuid_counts[man_obj.uuid]
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = geometry_type
geo.latitude = lon_lat[1]
geo.longitude = lon_lat[0]
geo.specificity = 0
# dump coordinates as json string
geo.coordinates = coord_str
try:
geo.save()
except:
print('Problem saving: ' + str(man_obj.uuid))
quit()
def create_geospatial_data_from_geonames_json(self, manifest, json_data):
""" creates and saves geospatial data derived from geonames """
ok = False
coords = ''
lat = False
lon = False
ftype = 'Point'
if 'bbox' in json_data:
ftype = 'Polygon'
east = json_data['bbox']['east']
south = json_data['bbox']['south']
west = json_data['bbox']['west']
north = json_data['bbox']['north']
lat = (south + north) / 2
lon = (east + west) / 2
coordinates = []
cood_inner = []
cood_inner.append([east, south])
cood_inner.append([east, north])
cood_inner.append([west, north])
cood_inner.append([west, south])
cood_inner.append([east, south])
coordinates.append(cood_inner)
coords = json.dumps(coordinates, indent=4, ensure_ascii=False)
else:
coords = ''
if 'lat' in json_data:
lat = float(json_data['lat'])
if 'lng' in json_data:
lon = float(json_data['lng'])
if lat is not False and lon is not False:
# we have OK coodinates, let's save them
geodata = Geospace()
geodata.uuid = manifest.uuid
geodata.project_uuid = manifest.project_uuid
geodata.source_id = 'geonames-api'
geodata.item_type = manifest.item_type
geodata.feature_id = 1
geodata.meta_type = 'oc-gen:discovey-location'
geodata.ftype = ftype
geodata.latitude = lat
geodata.longitude = lon
geodata.specificity = self.specifity
geodata.coordinates = coords
geodata.note = 'Location data from GeoNames.org'
geodata.save()
ok = True
if ok:
self.new_geodata += 1
output = 'New geodata [' + str(self.new_geodata) + '] ' + str(
manifest.label)
output += ' (' + str(manifest.uuid) + ')'
print(output)
return ok
def add_feature_to_existing_items(self, feature):
""" Process a feature to extract geospatial
object. It will:
(1) Find the appropriate item in the manifest table
(2) Adds a record in the geospace table
"""
man_obj = None
if 'properties' in feature:
props = feature['properties']
if 'uri' in props:
try_uuid = props['uri'].split('/')[-1]
man_objs = Manifest.objects.filter(uuid=try_uuid)[:1]
if man_objs:
man_obj = man_objs[0]
if man_obj and 'geometry' in feature:
# first get and validate the coordinates from the GeoJSON file
if man_obj.uuid not in self.uuid_counts:
self.uuid_counts[man_obj.uuid] = 0
self.uuid_counts[man_obj.uuid] += 1
geometry_type = feature['geometry']['type']
coordinates = feature['geometry']['coordinates']
v_geojson = ValidateGeoJson()
c_ok = v_geojson.validate_all_geometry_coordinates(
geometry_type, coordinates)
if not c_ok:
print('Fixing coordinates for: {}'.format(man_obj.uuid))
coordinates = v_geojson.fix_geometry_rings_dir(
geometry_type, coordinates)
if self.delete_old_geo and self.uuid_counts[man_obj.uuid] < 2:
Geospace.objects.filter(uuid=man_obj.uuid).delete()
coord_str = json.dumps(coordinates, indent=4, ensure_ascii=False)
gg = GeospaceGeneration()
lon_lat = gg.get_centroid_lonlat_coordinates(
coord_str, geometry_type)
print('Saving new geomettry for: ' + str(man_obj.uuid))
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = self.uuid_counts[man_obj.uuid]
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = geometry_type
geo.latitude = lon_lat[1]
geo.longitude = lon_lat[0]
geo.specificity = 0
# dump coordinates as json string
geo.coordinates = coord_str
try:
geo.save()
except:
print('Problem saving: ' + str(man_obj.uuid))
quit()
def make_geo_obj(self, feature_id, lon, lat, coords=False):
geo_obj = Geospace()
geo_obj.uuid = self.project_uuid
geo_obj.project_uuid = self.project_uuid
geo_obj.source_id = 'Project metadata summary'
geo_obj.item_type = 'projects'
geo_obj.feature_id = feature_id
geo_obj.meta_type = 'oc-gen:geo-coverage'
if coords is False:
geo_obj.ftype = 'Point'
geo_obj.coordinates = ''
else:
geo_obj.ftype = 'Polygon'
geo_obj.coordinates = json.dumps(coords, ensure_ascii=False)
geo_obj.latitude = lat
geo_obj.longitude = lon
geo_obj.specificity = 0
geo_obj.note = 'Project geographic coverage \
summarized from geospatial data \
describing subjects published \
with this project.'
return geo_obj
def make_geo_obj(self, feature_id, lon, lat, coords=False):
geo_obj = Geospace()
geo_obj.uuid = self.project_uuid
geo_obj.project_uuid = self.project_uuid
geo_obj.source_id = 'Project metadata summary'
geo_obj.item_type = 'projects'
geo_obj.feature_id = feature_id
geo_obj.meta_type = 'oc-gen:geo-coverage'
if coords is False:
geo_obj.ftype = 'Point'
geo_obj.coordinates = ''
else:
geo_obj.ftype = 'Polygon'
geo_obj.coordinates = json.dumps(coords, ensure_ascii=False)
geo_obj.latitude = lat
geo_obj.longitude = lon
geo_obj.specificity = self.project_specificity
geo_obj.note = 'Project geographic coverage \
summarized from geospatial data \
describing subjects published \
with this project.'
return geo_obj
def add_sites_geo_from_filecache(self):
""" updates Open Context to save new sites
location information
"""
if self.filecache is not None:
# print('Cache update !: ' + self.cache_filekey)
self.filecache.working_dir = self.working_dir
json_obj = self.filecache.get_dict_from_file(self.cache_filekey)
if isinstance(json_obj, dict):
if 'trinomial_refs' in json_obj:
for tri_ref in json_obj['trinomial_refs']:
for trinomial in tri_ref['trinomials']:
man_objs = Manifest.objects.filter(
label=trinomial, class_uri='oc-gen:cat-site')
if len(man_objs) > 0:
man_obj = man_objs[0]
county_code = trinomial[0:4]
if county_code in self.COUNTY_GEO:
county_geo = self.COUNTY_GEO[county_code]
print('County code: ' + county_code +
' is ' + str(county_geo))
geos = Geospace.objects.filter(
uuid=man_obj.uuid)[:1]
if len(geos) < 1:
geo = Geospace()
geo.uuid = man_obj.uuid
geo.project_uuid = man_obj.project_uuid
geo.source_id = self.source_id
geo.item_type = man_obj.item_type
geo.feature_id = 1
geo.meta_type = "oc-gen:discovey-location"
geo.ftype = 'Point'
geo.latitude = county_geo['lat']
geo.longitude = county_geo['lon']
geo.specificity = -11
geo.save()
def add_update_geo_data(self, post_data):
""" Updates a file associated with a media item """
ok = True
errors = []
note = ''
required_params = [
'source_id', 'hash_id', 'meta_type', 'ftype', 'feature_id',
'latitude', 'longitude', 'specificity', 'geojson'
]
for r_param in required_params:
if r_param not in post_data:
# we're missing some required data
# don't create the item
ok = False
message = 'Missing paramater: ' + r_param + ''
errors.append(message)
note = '; '.join(errors)
if ok:
coordinates = ''
hash_id = post_data['hash_id'].strip()
if len(hash_id) < 1:
hash_id = False
source_id = post_data['source_id'].strip()
ftype = post_data['ftype'].strip()
# validate the dates
output = self.validate_float_param('latitude', post_data, errors)
latitude = output['float']
errors = output['errors']
output = self.validate_float_param('longitude', post_data, errors)
longitude = output['float']
errors = output['errors']
output = self.validate_int_param('feature_id', post_data, errors)
feature_id = output['integer']
errors = output['errors']
output = self.validate_int_param('specificity', post_data, errors)
specificity = output['integer']
errors = output['errors']
# now check for GeoJSON, and validate if added
coordinates = ''
geojson_str = post_data['geojson'].strip()
if len(geojson_str) > 1:
# we have a geojson object
output = self.validate_geojson(geojson_str, errors)
coordinates = output['coordinates']
if output['type'] is not False:
ftype = output['type']
if isinstance(coordinates, str):
v_geojson = ValidateGeoJson()
coord_list = json.loads(coordinates)
coord_list = v_geojson.fix_geometry_rings_dir(
ftype, coord_list)
coordinates = json.dumps(coord_list)
gg = GeospaceGeneration()
lon_lat = gg.get_centroid_lonlat_coordinates(
coordinates, ftype)
latitude = lon_lat[1]
longitude = lon_lat[0]
if coordinates is False and latitude is not False\
and longitude is not False:
# no coordinates input in GeoJSON so make them
ftype = 'Point'
coordinates = json.dumps([longitude, latitude],
indent=4,
ensure_ascii=False)
if latitude is False or longitude is False \
or feature_id is False or coordinates is False \
or specificity is False:
# 1 or more of the geospatial data are bad
ok = False
note = '; '.join(errors)
else:
act_geo = None
if hash_id is not False:
try:
act_geo = Geospace.objects.get(hash_id=hash_id)
except Geospace.DoesNotExist:
act_geo = False
ok = False
message = 'Cannot find geo record for hash_id: ' + str(
hash_id)
errors.append(message)
note = '; '.join(errors)
if ok:
# we're OK to add to an event
if act_geo is None:
act_geo = Geospace()
act_geo.uuid = self.manifest.uuid
act_geo.project_uuid = self.manifest.project_uuid
act_geo.source_id = source_id
act_geo.meta_type = post_data['meta_type'].strip()
act_geo.ftype = ftype
act_geo.feature_id = feature_id
act_geo.latitude = latitude
act_geo.longitude = longitude
act_geo.coordinates = coordinates
act_geo.specificity = specificity
act_geo.save()
note = 'Updated geodata for ' + self.manifest.uuid
if ok:
# now clear the cache a change was made
cache.clear()
else:
self.errors = errors
self.response = {
'action': 'add-update-geo-data',
'ok': ok,
'change': {
'note': note
}
}
return self.response
def add_update_geo_data(self, post_data):
""" Updates a file associated with a media item """
ok = True
errors = []
note = ''
required_params = ['source_id',
'hash_id',
'meta_type',
'ftype',
'feature_id',
'latitude',
'longitude',
'specificity',
'geojson']
for r_param in required_params:
if r_param not in post_data:
# we're missing some required data
# don't create the item
ok = False
message = 'Missing paramater: ' + r_param + ''
errors.append(message)
note = '; '.join(errors)
if ok:
coordinates = ''
hash_id = post_data['hash_id'].strip()
if len(hash_id) < 1:
hash_id = False
source_id = post_data['source_id'].strip()
ftype = post_data['ftype'].strip()
# validate the dates
output = self.validate_float_param('latitude', post_data, errors)
latitude = output['float']
errors = output['errors']
output = self.validate_float_param('longitude', post_data, errors)
longitude = output['float']
errors = output['errors']
output = self.validate_int_param('feature_id', post_data, errors)
feature_id = output['integer']
errors = output['errors']
output = self.validate_int_param('specificity', post_data, errors)
specificity = output['integer']
errors = output['errors']
# now check for GeoJSON, and validate if added
coordinates = ''
geojson_str = post_data['geojson'].strip()
if len(geojson_str) > 1:
# we have a geojson object
output = self.validate_geojson(geojson_str, errors)
coordinates = output['coordinates']
if output['type'] is not False:
ftype = output['type']
if isinstance(coordinates, str):
v_geojson = ValidateGeoJson()
coord_list = json.loads(coordinates)
coord_list = v_geojson.fix_geometry_rings_dir(ftype, coord_list )
coordinates = json.dumps(coord_list)
gg = GeospaceGeneration()
lon_lat = gg.get_centroid_lonlat_coordinates(coordinates, ftype)
latitude = lon_lat[1]
longitude = lon_lat[0]
if coordinates is False and latitude is not False\
and longitude is not False:
# no coordinates input in GeoJSON so make them
ftype = 'Point'
coordinates = json.dumps([longitude, latitude],
indent=4,
ensure_ascii=False)
if latitude is False or longitude is False \
or feature_id is False or coordinates is False \
or specificity is False:
# 1 or more of the geospatial data are bad
ok = False
note = '; '.join(errors)
else:
act_geo = None
if hash_id is not False:
try:
act_geo = Geospace.objects.get(hash_id=hash_id)
except Geospace.DoesNotExist:
act_geo = False
ok = False
message = 'Cannot find geo record for hash_id: ' + str(hash_id)
errors.append(message)
note = '; '.join(errors)
if ok:
# we're OK to add to an event
if act_geo is None:
act_geo = Geospace()
act_geo.uuid = self.manifest.uuid
act_geo.project_uuid = self.manifest.project_uuid
act_geo.source_id = source_id
act_geo.meta_type = post_data['meta_type'].strip()
act_geo.ftype = ftype
act_geo.feature_id = feature_id
act_geo.latitude = latitude
act_geo.longitude = longitude
act_geo.coordinates = coordinates
act_geo.specificity = specificity
act_geo.save()
note = 'Updated geodata for ' + self.manifest.uuid
if ok:
# now clear the cache a change was made
cache.clear()
else:
self.errors = errors
self.response = {'action': 'add-update-geo-data',
'ok': ok,
'change': {'note': note}}
return self.response
def process_geojson_item(self,
subject_field_num,
subject_in_rows,
subject_uuid):
""" adds geojson data if it exists for an item
"""
if subject_field_num in self.geojson_rels:
geojson_field_num = self.geojson_rels[subject_field_num]
geojson_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=geojson_field_num,
row_num__in=subject_in_rows)\
.exclude(record='')
# the rows object now has a list of all the rows, with validated coordinate
# data. Now we can add these to the database!
geo_keys_done = []
gg = GeospaceGeneration()
for geojson_rec in geojson_recs:
geo_key = geojson_rec.rec_hash
record = geojson_rec.record
if geo_key not in geo_keys_done:
geo_keys_done.append(geo_key)
try:
json_obj = json.loads(record)
except:
json_obj = False
if isinstance(json_obj, dict):
if 'type' in json_obj \
and 'coordinates' in json_obj:
# OK we've got good data
lon_lat = gg.get_centroid_lonlat_coordinates(record,
json_obj['type'])
if isinstance(lon_lat, tuple):
# print('Good centroid for ' + str(subject_uuid))
# great! We have centroid lon_lat
# this is another data validation passed
coordinates = json_obj['coordinates']
same_geos = Geospace.objects\
.filter(uuid=subject_uuid,
coordinates=coordinates)[:1]
if len(same_geos) < 1:
# OK it is valid and does not exist for the item
# we can add it
geo = Geospace()
geo.uuid = str(subject_uuid)
geo.project_uuid = self.project_uuid
geo.source_id = self.source_id
geo.item_type = 'subjects'
geo.feature_id = len(geo_keys_done)
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = json_obj['type']
geo.latitude = lon_lat[1]
geo.longitude = lon_lat[0]
geo.specificity = 0
# dump coordinates as json string
geo.coordinates = json.dumps(coordinates,
indent=4,
ensure_ascii=False)
try:
geo.save()
except:
print('Did not like ' + str(row) + ' with ' + str(subject_uuid))
quit()
def process_geospace_item(self,
subject_field_num,
subject_in_rows,
subject_uuid):
""" adds lat lon data if it exists for an item
"""
if subject_field_num in self.geospace_fields:
# this subject field has associated lat - lon data
# now make a list where of rows that have non-blank lat data
act_geo_fields = self.geospace_fields[subject_field_num]
rows = {}
lat_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=act_geo_fields['lat'],
row_num__in=subject_in_rows)\
.exclude(record='')
for lat_rec in lat_recs:
rows[lat_rec.row_num] = {'lat': False,
'lon': False}
rows[lat_rec.row_num]['lat'] = self.validate_geo_coordinate(lat_rec.record,
'lat')
lon_recs = ImportCell.objects\
.filter(source_id=self.source_id,
field_num=act_geo_fields['lon'],
row_num__in=subject_in_rows)\
.exclude(record='')
for lon_rec in lon_recs:
if lon_rec.row_num not in rows:
rows[lon_rec.row_num] = {'lat': False,
'lon': False}
rows[lon_rec.row_num]['lon'] = self.validate_geo_coordinate(lon_rec.record,
'lon')
# the rows object now has a list of all the rows, with validated coordinate
# data. Now we can add these to the database!
geo_feature = 1
geo_keys_done = []
for row_key, row in rows.items():
if row['lat'] is not False and \
row['lon'] is not False:
geo_key = str(row['lat']) + ',' + str(row['lon'])
if geo_key not in geo_keys_done:
# we havent checked this set of coordinates yet
geo_keys_done.append(geo_key)
# now check to make sure we don't already have these coordinates
# on this item
same_geos = Geospace.objects\
.filter(uuid=subject_uuid,
latitude=row['lat'],
longitude=row['lon'])[:1]
if len(same_geos) < 1:
# it is a new coordinate, ok to add
geo = Geospace()
geo.uuid = str(subject_uuid)
geo.project_uuid = self.project_uuid
geo.source_id = self.source_id
geo.item_type = 'subjects'
geo.feature_id = geo_feature
geo.meta_type = ImportFieldAnnotation.PRED_GEO_LOCATION
geo.ftype = 'Point'
geo.latitude = row['lat']
geo.longitude = row['lon']
geo.specificity = 0
# dump coordinates as json string in lon - lat (GeoJSON order)
geo.coordinates = json.dumps([row['lon'], row['lat']],
indent=4,
ensure_ascii=False)
try:
geo.save()
geo_feature += 1
except:
print('Did not like ' + str(row) + ' with ' + str(subject_uuid))
quit()