def get_location(address):
logger.info(f"Determining location of {address}")
geolocator = GeoNames(username=CONFIG["geonames_username"])
location = geolocator.geocode(address)
if location is None:
logger.error(f"Unable to find address \"{address}\"")
return location
def locate(request, country, city):
"""
Locates a city input by the user.
"""
dajax = Dajax()
g = GeoNames(None, "veitheller")
place, (lat, lon) = g.geocode(str(city) + ", " + str(country))
dajax.add_data([lat, lon], 'addLocationMarker')
return dajax.json()
def locate(request, country, city):
"""
Locates a city input by the user.
"""
dajax = Dajax()
g = GeoNames(None, "veitheller")
place, (lat, lon) = g.geocode(str(city) + ", " + str(country))
dajax.add_data([lat, lon], 'addLocationMarker')
return dajax.json()
def geopy_geolocate_by_address_using_geonames(query):
from geopy.geocoders import GeoNames
print 'Search by Geonames location, query: ', query
geolocator = GeoNames(username=GEONAMES_USER_NAME)
location = geolocator.geocode(query)
if location != None:
print 'Search by Geonames location, address: ', location.address
print 'Search by Geonames location, coordinates:', location.latitude, location.longitude
print 'Search by Geonames location, location raw:', location.raw
else:
print 'Search by Geonames location, location for query:', query, 'could not be found.'
return location
def extractLocation(self, location):
geolocator = GeoNames(username='******')
geolocation = geolocator.geocode(location, timeout=10000)
geopoint = {}
if geolocation == None:
return None;
else:
geopoint['latitude'] = geolocation.latitude
geopoint['longitude'] = geolocation.longitude
return geopoint
def load(request):
"""
Loads locations.
"""
dajax = Dajax()
apis = Api.objects.filter(user=User.objects.get(username=request.user))
api_objects = []
if not apis:
error = "'Could not load data. No API available.'"
dajax.script("toastr.warning(" + error + ", 'API warning')")
dajax.script("markers = []")
return dajax.json()
plugindir = os.listdir(preferences.BASE_DIR + "/datavis/plugins")
for api in apis:
if not api.api + ".py" in plugindir:
error = "'Could not load API " + api.api + ". No such API.'"
dajax.script("toastr.error(" + error + ", 'API error')")
credentials = ApiKey.objects.filter(identification=api)
if api.needs_credentials and not credentials:
error = "'Could not load API " + api.api + ". No credentials.'"
dajax.script("toastr.error(" + error + ", 'API error')")
continue
impobj = getattr(
__import__("datavis.plugins." + api.api, fromlist=[api.api]),
api.api)
if credentials:
api_objects.append(
APIInterface(api.api, impobj, credentials[0].authentication))
else:
api_objects.append(APIInterface(api.api, impobj))
script = "markers = ["
g = GeoNames(None, "veitheller")
for api in api_objects:
for entry in api.locations:
entry_name = api.locations[entry][0] + ", " + entry
try:
place, (lat, lon) = g.geocode(entry_name)
except (TypeError, exc.GeopyError):
continue
script += str([lat, lon, entry_name]) + ","
script = script[:-1] + script[-1:].replace(",", "]")
dajax.script(script)
return dajax.json()
class GeoFinder(object):
PICKLED_GEO = "geomappings.pkl"
PICKLED_GEO_BLACKLIST = "geomappings_blacklist.pkl"
GEO_USER_NAME = "arssher"
def __init__(self):
self.geolocator = GeoNames(username=self.GEO_USER_NAME)
self.geomappings = {}
self.geomappings_blacklist = []
if os.path.isfile(self.PICKLED_GEO):
with open(self.PICKLED_GEO, 'rb') as f:
self.geomappings = pickle.load(f)
if os.path.isfile(self.PICKLED_GEO_BLACKLIST):
with open(self.PICKLED_GEO_BLACKLIST, 'rb') as f:
self.geomappings_blacklist = pickle.load(f)
log_info("known geomappings size: %s " % len(self.geomappings))
log_info("known geo blacklist size: %s " % len(self.geomappings_blacklist))
def __del__(self):
with open(self.PICKLED_GEO, 'wb') as f:
pickle.dump(self.geomappings, f, pickle.HIGHEST_PROTOCOL)
with open(self.PICKLED_GEO_BLACKLIST, 'wb') as f:
pickle.dump(self.geomappings_blacklist, f, pickle.HIGHEST_PROTOCOL)
def get_location(self, location_string):
if location_string in self.geomappings:
return self.geomappings[location_string]
elif location_string in self.geomappings_blacklist:
return (0, 0, "", 0)
else:
location = self.geolocator.geocode(location_string, exactly_one=True, timeout=60)
if location and u'countryCode' in location.raw:
cc_alphabet = location.raw[u'countryCode'].encode('utf_8')
cc_numeric = int(countries.get(cc_alphabet).numeric)
res = (location.latitude, location.longitude, location.raw[u'countryName'].encode('utf_8'), cc_numeric)
self.geomappings[location_string] = res
if len(self.geomappings) % 200 == 0:
log_info("Geomappings size now %s" % len(self.geomappings))
return res
else:
self.geomappings_blacklist.append(location_string)
log_warn("Failed to get location for string %s" % location_string.encode('utf_8'))
return (0, 0, "", 0)
def load(request):
"""
Loads locations.
"""
dajax = Dajax()
apis = Api.objects.filter(user=User.objects.get(username=request.user))
api_objects = []
if not apis:
error = "'Could not load data. No API available.'"
dajax.script("toastr.warning(" + error + ", 'API warning')")
dajax.script("markers = []")
return dajax.json()
plugindir = os.listdir(preferences.BASE_DIR + "/datavis/plugins")
for api in apis:
if not api.api + ".py" in plugindir:
error = "'Could not load API " + api.api + ". No such API.'"
dajax.script("toastr.error(" + error + ", 'API error')")
credentials = ApiKey.objects.filter(identification=api)
if api.needs_credentials and not credentials:
error = "'Could not load API " + api.api + ". No credentials.'"
dajax.script("toastr.error(" + error + ", 'API error')")
continue
impobj = getattr(__import__("datavis.plugins." + api.api,
fromlist=[api.api]),
api.api)
if credentials:
api_objects.append(APIInterface(api.api, impobj,
credentials[0].authentication))
else:
api_objects.append(APIInterface(api.api, impobj))
script = "markers = ["
g = GeoNames(None, "veitheller")
for api in api_objects:
for entry in api.locations:
entry_name = api.locations[entry][0] + ", " + entry
try:
place, (lat, lon) = g.geocode(entry_name)
except (TypeError, exc.GeopyError):
continue
script += str([lat, lon, entry_name]) + ","
script = script[:-1] + script[-1:].replace(",", "]")
dajax.script(script)
return dajax.json()
def extract_coord_list(listemon):
"""
Extraction des coordonnées des monuments recherchés.
arg : liste des monuments choisis
Retourne : liste de tuples (monument, latitude, longitude)
"""
geolocatorOSM = Nominatim() #Open Street Maps
geolocatorGN = GeoNames(username="******")
prob = ["Hôtel de Ville", "Pont Neuf", "Place de la Concorde"
] #obtention de coordonnees erronnées avec Open Street Maps
mon_coord = []
for monument in listemon:
if monument not in prob:
location = geolocatorOSM.geocode(
monument) #coordonnées avec Open Street Maps
mon_coord.append((monument, location.latitude, location.longitude))
else:
location = geolocatorGN.geocode(
monument) #coordonnées avec GeoNames
mon_coord.append((monument, location.latitude, location.longitude))
return mon_coord
def geocoding_cities():
geolocator = GeoNames(username='******')
# output file
fw = codecs.open('../city_coords_AllCategories.csv','w','utf-8')
#read from the city list
with codecs.open('../city_poi_by_type/AllCategory/city_list.csv','r','utf-8') as fr:
for line in fr:
this_city_name = line.strip()
splitted = this_city_name.split(",")
this_city_name_new = splitted[0].title()+", "+splitted[1].upper()
location = geolocator.geocode(this_city_name_new)
if location != None:
#print(location.raw)
#location = json.dumps(location.raw, encoding='utf-8')
print(this_city_name +": "+ str(location.latitude) +", "+ str(location.longitude))
fw.write(this_city_name+"|"+str(location.latitude)+"|"+str(location.longitude)+"\n")
else:
print(this_city_name+' not geocoded')
fw.write(this_city_name+"||\n")
fw.close()
def geoLocate(list_of_places, list_of_locations):
#Using Geopy for geolocations NOTE this works
GeoNamesAccounts = [
"semantic_1", "semantic_2", "semantic_3", "semantic_4", "semantic_5",
"semantic_6"
]
latin = '^[ -~]+$'
holder = []
holder = holder + GeoNamesAccounts
geolocations = []
choice = random.choice(GeoNamesAccounts)
GeoNamesAccounts.remove(choice)
geolocator = GeoNames(username=choice)
#print("After geoLocate Settings") #locational information printed in console to know where the program is getting caught
#removing duplicates to be sure it should already be distinct
places = list(set(list_of_places))
#print(len(places))
# i = 0
for place in places:
if (len(place) == 1) and (place.lower() == "island"
or place.lower() == "islands"):
pass
# i += 1
geo = None
#print("new place loop") #locational information printed in console to know where the program is getting caught
# if counter >= 1500: #Code for when we didn't have multiple locations from the config file
# try:
# choice = random.choice(GeoNamesAccounts)
# except:
# GeoNamesAccounts = holder + GeoNamesAccounts
# choice = random.choice(GeoNamesAccounts)
# GeoNamesAccounts.remove(choice)
# geolocator = GeoNames(username=choice)
# counter = 1
while geo == None:
try:
#print("in geolocator loop")
geo = geolocator.geocode(place[0], timeout=20)
if geo != None:
if re.search(latin,
geo.address): #is it an latin letter location
if 3 > len(geo.address):
geolocations.append(geo)
break
for location in list_of_locations:
#print(geo.address)
#print(location+"\n")
split_loc = geo.address.split(
" ") #len is for country names or state names
if location in geo.address or 0 < len(
split_loc) < 3:
geolocations.append(geo)
break
elif not location in geo.address:
pass
break
while True: #continue till all locations are exhausted
#print("going through location loop") #locational information printed in console to know where the program is getting caught
if not "Montana" in list_of_locations and not "Washington" in list_of_locations:
break #NOTE this is to avoid increasing the time complexity too much, since he's a Montana authour most of his work will feature town names in Montana and Washington
new_place = place[0] + " " + list_of_locations[0]
#print(new_place)
#print(place[0])
#print(list_of_locations[0])
try:
geo = geolocator.geocode(new_place, timeout=20)
#print(geo)
break
except Exception as e:
#print(type(e))
if "limit" in e:
geoResetter(
GeoNamesAccounts, holder
) #need to switch account because lookup limit has been reached
continue
if geo != None:
for location in list_of_locations:
#print(geo.address)
#print(location+"\n")
if location in geo.address:
geolocations.append(geo)
break
elif not location in geo.address:
pass
break
except Exception as e:
print(e)
geoResetter(
GeoNamesAccounts, holder
) #need to switch account because lookup limit has been reached
continue
# Used to error check looping to make sure the same amount of iterations were happening as there were places
# print(len(places))
# print(str(i)+" interations")
geoplaces = []
#print(geolocations)
for geoloc in geolocations:
geoplaces.append(geoloc.address + " (" + str(geoloc.latitude) + "," +
str(geoloc.longitude) + ")")
geolocations = KeywordCounter(geoplaces)
return geolocations
elif len(k) == 6:
return (int(k[0]) + (int(k[1]) + float(k[2]) / 60) / 60,
int(k[3]) + (int(k[4]) + float(k[5]) / 60) / 60)
else:
print('Wrong format')
return manual(inpt)
for i in sopron:
print(i)
k = i[0].split(", ")
locations = []
for j in k:
j1 = j.replace('-', '')
location = geolocator.geocode(j1.rstrip(),
False,
country=['hu', 'at', 'si', 'sk'])
if location != None:
locations.extend(location)
else:
if j1.find('(') != -1:
j2 = j1.split('(')
j3 = j2[-1].replace(')', '').replace('-', '').rstrip() + j2[0]
location = geolocator.geocode(j3.rstrip(),
False,
country=['hu', 'at', 'si', 'sk'])
if location != None:
locations.extend(location)
validlocs = []
subvalidlocs = []
for j1 in range(len(locations)):
def geocode(string):
geolocator = GeoNames(username='******')
return geolocator.geocode(string)
class GeocodingEmDat(object):
def __init__(self,paese):
self.paese = paese
self.historical_table = "c:/data/tools/sparc/input_data/historical_data/floods - refine.csv"
self.geolocator = Nominatim()
self.geolocator_geonames = GeoNames(country_bias=self.paese, username='******', timeout=1)
self.outDriver = ogr.GetDriverByName("ESRI Shapefile")
self.countries_shp_location = os.getcwd() + '/input_data/countries'
self.outShp = os.getcwd() + "/input_data/geocoded/shp/" + self.paese + ".shp"
self.events_location = os.getcwd() + '/input_data/geocoded/shp/'
self.risk_map_location = os.getcwd() + '/input_data/geocoded/risk_map/'
def geolocate_accidents(self):
accidents = {}
with open(self.historical_table, 'rb') as csvfile:
luoghi_splittati = csv.reader(csvfile, delimiter=",", quotechar='"')
for row in luoghi_splittati:
if row[2] == self.paese:
id_incidente = str(row[9])
accidents[id_incidente] = {}
accidents[id_incidente]['paese'] = str(row[2])
accidents[id_incidente]['killed'] = str(row[6])
accidents[id_incidente]['affected'] = str(row[7])
accidents[id_incidente]['locations'] = {}
gruppo = str(row[3]).split(",")
quante_locations = len(gruppo)
for i in range(0, quante_locations):
accidents[id_incidente]['locations'][i] = gruppo[i].strip()
totali = 0
successo = 0
insuccesso = 0
geocoding_success_file = "C:/data/tools/sparc/input_data/geocoded/text/" + self.paese + ".txt"
geocoding_fail_file = "C:/data/tools/sparc/input_data/geocoded/text/" + self.paese + "_fail.txt"
# Control if accidents have been geocoded already
if os.path.exists(geocoding_success_file):
return "Geocoded already!!"
pass
else:
geocoding_success = open(geocoding_success_file, "wb+")
geocoding_fail = open(geocoding_fail_file, "wb+")
geocoding_success.write("id,lat,lon\n")
geocoding_fail.write("id,lat,lon\n")
try:
for incidente in accidents.iteritems():
for location_non_geocoded in incidente[1]['locations'].iteritems():
totali += 1
posto_attivo = location_non_geocoded[1]
if posto_attivo != 'NoData':
try:
print("Geocoding " + posto_attivo)
#location_geocoded = self.geolocator.geocode(posto_attivo, timeout=30)
location_geocoded = self.geolocator_geonames.geocode(posto_attivo,timeout=30)
if location_geocoded:
scrittura = posto_attivo + "," + str(location_geocoded.longitude) + "," + str(location_geocoded.latitude) + "\n"
geocoding_success.write(scrittura)
successo += 1
else:
geocoding_fail.write(posto_attivo + "," + str(0) + "," + str(0) + "\n")
insuccesso += 1
except ValueError as e:
print e.message
print "Total of %s events with %s successful %s unsuccessful and %d NULL" % (
str(totali), str(successo), str(insuccesso), (totali - successo - insuccesso))
perc = float(successo) / float(totali) * 100.0
print "Percentage %.2f of success" % perc
except:
print "No response from geocoding server"
pass
def create_validated_coords(self):
def calc_poligono_controllo():
poligono = sf.bbox
global poligono_controllo
poligono_controllo = ((poligono[2],poligono[1]), (poligono[2],poligono[3]), (poligono[0],poligono[3]), (poligono[0],poligono[1]))
global n
n = len(poligono_controllo)
def punti_dentro_poligono_di_controllo(x,y):
inside = False
p1x, p1y = poligono_controllo[0]
for i in range(n + 1):
p2x, p2y = poligono_controllo[i % n]
if y > min(p1y, p2y):
if y <= max(p1y, p2y):
if x <= max(p1x, p2x):
if p1y != p2y:
xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
if p1x == p2x or x <= xinters:
inside = not inside
p1x, p1y = p2x, p2y
return inside
def extract_country_shp():
# Get the input Layer
inShapefile = "input_data/gaul/gaul_wfp.shp"
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(inShapefile, 0)
inLayer = inDataSource.GetLayer()
inLayer.SetAttributeFilter("ADM0_NAME = '" + self.paese + "'")
# Create the output LayerS
outShapefile = "input_data/countries/" + self.paese + ".shp"
outDriver = ogr.GetDriverByName("ESRI Shapefile")
# Remove output shapefile if it already exists
if os.path.exists(outShapefile):
outDriver.DeleteDataSource(outShapefile)
# Create the output shapefile
outDataSource = outDriver.CreateDataSource(outShapefile)
out_lyr_name = os.path.splitext(os.path.split(outShapefile)[1])[0]
outLayer = outDataSource.CreateLayer(out_lyr_name, geom_type=ogr.wkbMultiPolygon)
# Add input Layer Fields to the output Layer if it is the one we want
inLayerDefn = inLayer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
# Get the output Layer's Feature Definition
outLayerDefn = outLayer.GetLayerDefn()
# Add features to the ouput Layer
for inFeature in inLayer:
# Create output Feature
outFeature = ogr.Feature(outLayerDefn)
# Add field values from input Layer
for i in range(0, outLayerDefn.GetFieldCount()):
fieldDefn = outLayerDefn.GetFieldDefn(i)
outFeature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), inFeature.GetField(i))
# Set geometry as centroid
geom = inFeature.GetGeometryRef()
outFeature.SetGeometry(geom.Clone())
# Add new feature to output Layer
outLayer.CreateFeature(outFeature)
# Close DataSources
inDataSource.Destroy()
outDataSource.Destroy()
dentro = 0
fuori = 0
coords_check_file_in = "input_data/geocoded/text/" + self.paese + ".txt"
coords_validated_file_out = str('input_data/geocoded/csv/' + str(self.paese) + '.csv')
if os.path.exists("input_data/countries/" + self.paese + ".shp"):
sf = shapefile.Reader("input_data/countries/" + str(self.paese).lower() + ".shp")
calc_poligono_controllo()
else:
extract_country_shp()
sf = shapefile.Reader("input_data/countries/" + str(self.paese).lower() + ".shp")
calc_poligono_controllo()
with open(coords_check_file_in) as csvfile_in:
lettore_comma = csv.reader(csvfile_in, delimiter=",", quotechar='"')
next(lettore_comma)
with open(coords_validated_file_out, 'wb') as csvfile_out:
scrittore = csv.writer(csvfile_out, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
intestazioni = "id", "lat", "lon"
scrittore.writerow(intestazioni)
for row in lettore_comma:
if (punti_dentro_poligono_di_controllo(float(row[1]), float(row[2]))):
stringa = str(row[0]), str(row[1]), str(row[2])
scrittore.writerow(stringa)
dentro += 1
else:
fuori += 1
csvfile_out.close()
csvfile_in.close()
#print "dentro %d" % dentro, "fuori %d" % fuori
def creazione_file_shp(self):
# Remove output shapefile if it already exists
if os.path.exists(self.outShp):
self.outDriver.DeleteDataSource(self.outShp)
#Set up blank lists for data
x, y, nomeloc=[], [], []
#read data from csv file and store in lists
with open('input_data/geocoded/csv/'+str(self.paese) + '.csv', 'rb') as csvfile:
r = csv.reader(csvfile, delimiter=';')
for i, row in enumerate(r):
if i > 0: #skip header
divisa = row[0].split(",")
#print divisa[0]
nomeloc.append(divisa[0])
x.append(float(divisa[1]))
y.append(float(divisa[2]))
#date.append(''.join(row[1].split('-')))#formats the date correctly
#target.append(row[2])
#Set up shapefile writer and create empty fields
w = shapefile.Writer(shapefile.POINT)
w.autoBalance = 1 #ensures gemoetry and attributes match
w.field('ID','N')
w.field('location','C', 50)
# w.field('Date','D')
# w.field('Target','C',50)
# w.field('ID','N')
#loop through the data and write the shapefile
for j,k in enumerate(x):
w.point(k,y[j]) #write the geometry
w.record(k,nomeloc[j]) #write the attributes
#Save shapefile
w.save(self.outShp)
def plot_mappa(self):
def GetExtent(gt,cols,rows):
ext=[]
xarr=[0, cols]
yarr=[0, rows]
for px in xarr:
for py in yarr:
x=gt[0]+(px*gt[1])+(py*gt[2])
y=gt[3]+(px*gt[4])+(py*gt[5])
ext.append([x,y])
#print x,y
yarr.reverse()
return ext
pathToRaster = "input_data/geocoded/risk_map/" + self.paese + ".tif"
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import numpy as np
from osgeo import gdal
raster = gdal.Open(pathToRaster, gdal.GA_ReadOnly)
array = raster.GetRasterBand(1).ReadAsArray()
msk_array = np.ma.masked_equal(array, value=65535)
# print 'Raster Projection:\n', raster.GetProjection()
geotransform = raster.GetGeoTransform()
cols = raster.RasterXSize
rows = raster.RasterYSize
ext = GetExtent(geotransform, cols, rows)
#print ext[1][0], ext[1][1]
#print ext[3][0], ext[3][1]
#map = Basemap(projection='merc',llcrnrlat=-80, urcrnrlat=80, llcrnrlon=-180,urcrnrlon=180,lat_ts=20,resolution='c')
map = Basemap(projection='merc', llcrnrlat=ext[1][1], urcrnrlat=ext[3][1], llcrnrlon=ext[1][0], urcrnrlon=ext[3][0],lat_ts=20, resolution='c')
# Add some additional info to the map
map.drawcoastlines(linewidth=1.3, color='white')
#map.drawrivers(linewidth=.4, color='white')
map.drawcountries(linewidth=.75, color='white')
#datain = np.flipud(msk_array)
datain = np.flipud(msk_array)
map.imshow(datain)#,origin='lower',extent=[ext[1][0], ext[3][0],ext[1][1],ext[3][1]])
plt.show()
def add_prj(self):
env.workspace = self.events_location
inData = self.paese + ".shp"
print "Proietto " + inData
try:
coordinateSystem = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]"
arcpy.DefineProjection_management(inData, coordinateSystem)
except arcpy.ExecuteError:
print arcpy.GetMessages(2)
arcpy.AddError(arcpy.GetMessages(2))
except Exception as e:
print e.args[0]
arcpy.AddError(e.args[0])
def create_heat_map(self):
# Local variables:
event_file_shp = self.events_location + self.paese + ".shp"
krn_map_file = self.risk_map_location + self.paese + ".tif"
try:
# Process: Kernel Density
arcpy.gp.KernelDensity_sa(event_file_shp, "NONE", krn_map_file, "0.02", "", "SQUARE_MAP_UNITS")
except arcpy.ExecuteError:
print "Errore" + self.paese
print arcpy.GetMessages(2)
arcpy.AddError(arcpy.GetMessages(2))
except Exception as e:
print "Exception " + self.paese
print e.args[0]
arcpy.AddError(e.args[0])
class GeoLocate(pygeoip.GeoIP):
"""
Geolocation class that inherits from GeoIP.
It also has an interface to geopy which
all in all seems a bit hacky to me.
You decide.
"""
def __init__(self, filename, geo_identity=None):
"""
Initializes the class.
Keyword arguments:
filename -- string representation of the file containing the geodata
geo_identity -- string representation of the identity in geonames
"""
pygeoip.GeoIP.__init__(self, filename)
self._setup_segments()
if geo_identity:
self.gnames = GeoNames(None, geo_identity)
def getOwnAddress(self):
"""Gets own address based on the IP Address of the user."""
self.address = str(self._getIPAddress())[2:-1]
(self.country, self.city, self.lat,
self.lon) = self.coordsFromAddr(*self._locateAddress())
def _getIPAddress(self):
"""
Gets own IP address using a web service. Is that too sloppy?
Returns:
string -- IP address
"""
return request.urlopen("http://bot.whatismyipaddress.com/").read()
def _locateAddress(self):
"""
Gets the city and country name for a certain location.
Returns:
Tuple of:
string -- city name
string -- country name
"""
return (self.record_by_addr(self.address)['city'],
self.record_by_addr(self.address)['country_name'])
def coordsFromAddr(self, cityname, countryname):
"""
Gets the coordinates for a certain city name.
Currently problematic with Korea; geonames
seems to have issues in that area.
Keyword Arguments:
cityname -- the name of the city searched for
countryname -- the name of the country searched for
Returns:
List of:
countryname -- country name provided
cityname -- city name provided
lat -- latitude of location (default None)
lon -- longitude of location (default None)
"""
try:
place, (lat, lon) = self.gnames.geocode(
str(cityname) + ", " + str(countryname))
return [countryname, cityname, lat, lon]
except (TypeError, exc.GeopyError):
return [countryname, cityname, None, None]
def ownCoordsFromAddr(self, cityname, countryname):
"""
Gets the coordinates for the own city name
and makes the place found the own location.
Currently problematic with Korea; geonames
seems to have issues in that area.
Keyword Arguments:
cityname -- the name of the city searched for
countryname -- the name of the country searched for
Returns:
Boolean -- Indicates whether the coordinates could be parsed
"""
self.city = cityname
self.country = countryname
try:
place, (lat, lon) = self.gnames.geocode(
str(cityname) + ", " + str(countryname))
self.lat = lat
self.lon = lon
return True
except (TypeError, exc.GeopyError):
self.lat = None
self.lon = None
return False
def lookForDup(self, location_list, location, treshold):
"""
Searches for points to be merged in a list of locations
within a certain treshold.
Keyword Arguments:
location_list -- list of known locations
location -- the location to be tested
treshold -- the treshold for the test
Returns:
Tuple of:
- Boolean -- Indicates whether the location could be merged
- ind -- Index of the location which fits (default None)
Known Exceptions:
Value Error:
Raised when gnames does not know the location or is not set
"""
if not self.gnames:
raise ValueError
try:
loc_tuple = Point(location[2], location[3])
d = distance.distance
ind = 0
for test in location_list:
test_tuple = Point(test[2], test[3])
if float(d(loc_tuple, test_tuple).miles) < treshold:
return False, ind
ind += 1
return True, None
except ValueError:
return True, None
#import module geopy
from geopy.geocoders import GeoNames
import json
geolocator = GeoNames(username='******') # Register at Geonames
with open('places.csv','r', encoding='utf-8') as f:
for line in f:
for word in line.split():
location = geolocator.geocode(word, timeout=10)
if location != None:
json.dumps(location.raw)
print (word,"\t",location.latitude,",",location.longitude,sep='')
else:
print (word)
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 17 12:01:10 2019
@author: LokalAdm
"""
from geopy.geocoders import GeoNames
import json
geolocator = GeoNames(username='******') # Register at Geonames
location = geolocator.geocode("冥府", timeout=10)
if location != None:
# json.dump(location.raw, f)
print(location)
else:
print("No location!", location)
instance = GeoNames(country_bias="PK", username="", timeout=15)
with open('bank_tehsils.csv', 'rb') as csvfile:
spamreader = csv.reader(csvfile, delimiter=',')
rownum = 0
array = []
for row in spamreader:
if rownum == 0:
header = row
else:
try:
print("geocoding " + str(rownum))
place = row[1]
# Geocode based on name
location = GeoNames.geocode(instance, place, timeout=15)
if hasattr(location, "latitude"):
array.append(
[row[0], place, location.latitude, location.longitude])
print("succesfully completed " + place)
else:
print("couldn't find " + place)
array.append([row[0], place, "NA", "NA"])
except geopy.exc.GeopyError as err:
print(err)
print("bad request for " + place)
array.append([row[0], place, "error", "error"])
rownum += 1
csvfile.close()
with open('tehsils_geonames.csv', 'wb') as outfile:
class UtilitieSparc(ProjectFlood):
campo_nome_paese = "ADM0_NAME"
campo_iso_paese = "ADM0_CODE"
campo_nome_admin = "ADM2_NAME"
campo_iso_admin = "ADM2_CODE"
nome_paese = ""
cod_paese = ""
def __init__(self, area,schema,tabella_pesi,tabella_pop_stat,tabella_cicloni):
self.dati_per_plot = {}
self.dati_per_prob = {}
self.geolocator = Nominatim()
self.geolocator_geonames = GeoNames(country_bias=self.paese, username='******', timeout=1)
self.outDriver = ogr.GetDriverByName("ESRI Shapefile")
self.outShp = "classes/geocodifica/shp/" + self.paese + ".shp"
self.area = area
self.schema = schema
self.tabella_pesi = tabella_pesi
self.tabella_pop_stat = tabella_pop_stat
self.tabella_cicloni = tabella_cicloni
try:
self.conn = psycopg2.connect("dbname=sparc user=postgres")
except Exception as e:
print e.message
self.cur = self.conn.cursor()
def lista_admin0(self):
numFeatures = self.layer.GetFeatureCount()
lista_stati = []
for featureNum in range(numFeatures):
feature = self.layer.GetFeature(featureNum)
nome_paese = feature.GetField(self.campo_nome_paese)
lista_stati.append(nome_paese)
seen = set()
seen_add = seen.add
lista_pulita = [x for x in lista_stati if not (x in seen or seen_add(x))]
lista_admin0 = sorted(lista_pulita)
return tuple(lista_admin0)
def lista_admin2(self, country):
country_capitalized = country.capitalize()
self.layer.SetAttributeFilter(self.campo_nome_paese + " = '" + country_capitalized + "'")
listone={}
lista_iso = []
lista_clean = []
lista_admin2 = []
for feature in self.layer:
cod_admin = feature.GetField(self.campo_iso_admin)
nome_zozzo = feature.GetField(self.campo_nome_admin)
unicode_zozzo = nome_zozzo.decode('utf-8')
nome_per_combo = unicodedata.normalize('NFKD', unicode_zozzo)
no_dash = re.sub('-', '_', nome_zozzo)
no_space = re.sub(' ', '', no_dash)
no_slash = re.sub('/', '_', no_space)
no_apice = re.sub('\'', '', no_slash)
no_bad_char = re.sub(r'-/\([^)]*\)', '', no_apice)
unicode_pulito = no_bad_char.decode('utf-8')
nome_pulito = unicodedata.normalize('NFKD', unicode_pulito).encode('ascii', 'ignore')
lista_iso.append(cod_admin)
lista_clean.append(nome_pulito)
lista_admin2.append(nome_per_combo)
for i in range(len(lista_iso)):
listone[lista_iso[i]] = {'name_orig': lista_admin2[i],'name_clean':lista_clean[i]}
return lista_admin2, listone
def creazione_struttura(self, admin_global):
# Check in data structures exists and in case not create the directory named
# after the country and all the directories
UtilitieSparc.proj_dir
os.chdir(UtilitieSparc.proj_dir)
country_low = str(self.paese).lower()
if os.path.exists(country_low):
os.chdir(UtilitieSparc.proj_dir + country_low)
admin_low = str(self.admin).lower()
if os.path.exists(admin_low):
pass
else:
os.mkdir(admin_low)
os.chdir(UtilitieSparc.proj_dir + country_low + "/" + admin_global + "/")
os.mkdir("out")
else:
os.chdir(UtilitieSparc.proj_dir)
os.mkdir(country_low)
os.chdir(UtilitieSparc.proj_dir + country_low)
admin_low = str(self.admin).lower()
if os.path.exists(admin_low):
pass
else:
os.mkdir(admin_low)
os.chdir(UtilitieSparc.proj_dir + country_low + "/" + admin_global + "/")
os.mkdir("out")
return "Project created......\n"
def create_template(self,dir_template):
os.chdir(dir_template)
tabella = dbf.Table('monthly_values', 'month C(10);mean N(5,1)')
return tabella.filename
def geolocate_accidents(self):
accidents = {}
with open(self.historical_table, 'rb') as csvfile:
luoghi_splittati = csv.reader(csvfile, delimiter=",", quotechar='"')
for row in luoghi_splittati:
if row[2] == self.paese:
id_incidente = str(row[9])
accidents[id_incidente] = {}
accidents[id_incidente]['paese'] = str(row[2])
accidents[id_incidente]['killed'] = str(row[6])
accidents[id_incidente]['affected'] = str(row[7])
accidents[id_incidente]['locations'] = {}
gruppo = str(row[3]).split(",")
quante_locations = len(gruppo)
for i in range(0, quante_locations):
accidents[id_incidente]['locations'][i] = gruppo[i].strip()
totali = 0
successo = 0
insuccesso = 0
geocoding_success_file = "classes/geocodifica/text/" + self.paese + ".txt"
geocoding_fail_file = "classes/geocodifica/text/" + self.paese + "_fail.txt"
# Control if accidents have been geocoded already
if os.path.exists(geocoding_success_file):
print "Geocoded already!!"
pass
else:
geocoding_success = open(geocoding_success_file, "wb+")
geocoding_fail = open(geocoding_fail_file, "wb+")
geocoding_success.write("id,lat,lon\n")
geocoding_fail.write("id,lat,lon\n")
try:
for incidente in accidents.iteritems():
for location_non_geocoded in incidente[1]['locations'].iteritems():
totali += 1
posto_attivo = location_non_geocoded[1]
if posto_attivo != 'NoData':
try:
print("Geocoding " + posto_attivo)
#location_geocoded = self.geolocator.geocode(posto_attivo, timeout=30)
location_geocoded = self.geolocator_geonames.geocode(posto_attivo,timeout=30)
if location_geocoded:
scrittura = posto_attivo + "," + str(location_geocoded.longitude) + "," + str(location_geocoded.latitude) + "\n"
geocoding_success.write(scrittura)
successo += 1
else:
geocoding_fail.write(posto_attivo + "," + str(0) + "," + str(0) + "\n")
insuccesso += 1
except ValueError as e:
print e.message
print "Total of %s events with %s successful %s unsuccessful and %d NULL" % (
str(totali), str(successo), str(insuccesso), (totali - successo - insuccesso))
perc = float(successo) / float(totali) * 100.0
print "Percentage %.2f of success" % perc
except:
print "No response from geocoding server"
pass
def create_validated_coords(self):
def calc_poligono_controllo():
poligono = sf.bbox
global poligono_controllo
poligono_controllo = ((poligono[2],poligono[1]), (poligono[2],poligono[3]), (poligono[0],poligono[3]), (poligono[0],poligono[1]))
global n
n = len(poligono_controllo)
def punti_dentro_poligono_di_controllo(x,y):
inside = False
p1x, p1y = poligono_controllo[0]
for i in range(n + 1):
p2x, p2y = poligono_controllo[i % n]
if y > min(p1y, p2y):
if y <= max(p1y, p2y):
if x <= max(p1x, p2x):
if p1y != p2y:
xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
if p1x == p2x or x <= xinters:
inside = not inside
p1x, p1y = p2x, p2y
return inside
def extract_country_shp():
# Get the input Layer
inShapefile = "C:/data/input_data/gaul_2014_2008_2/gaul_wfp.shp"
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(inShapefile, 0)
inLayer = inDataSource.GetLayer()
inLayer.SetAttributeFilter("ADM0_NAME = '" + self.paese + "'")
# Create the output LayerS
outShapefile = "C:/data/input_data/countries/" + self.paese + ".shp"
outDriver = ogr.GetDriverByName("ESRI Shapefile")
# Remove output shapefile if it already exists
if os.path.exists(outShapefile):
outDriver.DeleteDataSource(outShapefile)
# Create the output shapefile
outDataSource = outDriver.CreateDataSource(outShapefile)
out_lyr_name = os.path.splitext(os.path.split(outShapefile)[1])[0]
outLayer = outDataSource.CreateLayer(out_lyr_name, geom_type=ogr.wkbMultiPolygon)
# Add input Layer Fields to the output Layer if it is the one we want
inLayerDefn = inLayer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
# Get the output Layer's Feature Definition
outLayerDefn = outLayer.GetLayerDefn()
# Add features to the ouput Layer
for inFeature in inLayer:
# Create output Feature
outFeature = ogr.Feature(outLayerDefn)
# Add field values from input Layer
for i in range(0, outLayerDefn.GetFieldCount()):
fieldDefn = outLayerDefn.GetFieldDefn(i)
outFeature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), inFeature.GetField(i))
# Set geometry as centroid
geom = inFeature.GetGeometryRef()
outFeature.SetGeometry(geom.Clone())
# Add new feature to output Layer
outLayer.CreateFeature(outFeature)
# Close DataSources
inDataSource.Destroy()
outDataSource.Destroy()
dentro = 0
fuori = 0
coords_check_file_in = "classes/geocodifica/text/" + self.paese + ".txt"
coords_validated_file_out = str('classes/geocodifica/csv/' + str(self.paese) + '.csv')
if os.path.exists("C:/data/input_data/countries/" + self.paese + ".shp"):
sf = shapefile.Reader("C:/data/input_data/countries/" + str(self.paese).lower() + ".shp")
calc_poligono_controllo()
else:
extract_country_shp()
sf = shapefile.Reader("C:/data/input_data/countries/" + str(self.paese).lower() + ".shp")
calc_poligono_controllo()
with open(coords_check_file_in) as csvfile_in:
lettore_comma = csv.reader(csvfile_in, delimiter=",", quotechar='"')
next(lettore_comma)
with open(coords_validated_file_out, 'wb') as csvfile_out:
scrittore = csv.writer(csvfile_out, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
intestazioni = "id", "lat", "lon"
scrittore.writerow(intestazioni)
for row in lettore_comma:
if (punti_dentro_poligono_di_controllo(float(row[1]), float(row[2]))):
stringa = str(row[0]), str(row[1]), str(row[2])
scrittore.writerow(stringa)
dentro += 1
else:
fuori += 1
csvfile_out.close()
csvfile_in.close()
#print "dentro %d" % dentro, "fuori %d" % fuori
def creazione_file_shp(self):
# Remove output shapefile if it already exists
if os.path.exists(self.outShp):
self.outDriver.DeleteDataSource(self.outShp)
#Set up blank lists for data
x, y, nomeloc=[], [], []
#read data from csv file and store in lists
with open('classes/geocodifica/csv/'+str(self.paese) + '.csv', 'rb') as csvfile:
r = csv.reader(csvfile, delimiter=';')
for i, row in enumerate(r):
if i > 0: #skip header
divisa = row[0].split(",")
#print divisa[0]
nomeloc.append(divisa[0])
x.append(float(divisa[1]))
y.append(float(divisa[2]))
#date.append(''.join(row[1].split('-')))#formats the date correctly
#target.append(row[2])
#Set up shapefile writer and create empty fields
w = shapefile.Writer(shapefile.POINT)
w.autoBalance = 1 #ensures gemoetry and attributes match
w.field('ID','N')
w.field('location','C', 50)
# w.field('Date','D')
# w.field('Target','C',50)
# w.field('ID','N')
#loop through the data and write the shapefile
for j,k in enumerate(x):
w.point(k,y[j]) #write the geometry
w.record(k,nomeloc[j]) #write the attributes
#Save shapefile
w.save(self.outShp)
def cancella_tabella(self):
comando_delete_table = "DROP TABLE " + self.schema + "." + self.tabella_pesi + " CASCADE;"
try:
self.cur.execute(comando_delete_table)
return "Table deleted"
except psycopg2.Error as delErrore:
errore_delete_tabella = delErrore.pgerror
return errore_delete_tabella
def crea_tabella(self):
try:
comando = "CREATE TABLE " + self.schema + "." + self.tabella_pesi + " (id serial PRIMARY KEY,month integer,weight double precision);"
print comando
self.cur.execute(comando)
#comando = "CREATE TABLE " + self.schema + "." + self.tabella_pesi + " (id serial PRIMARY KEY,month integer,weight double precision);"
#self.cur.execute(comando)
return "Table created"
except psycopg2.Error as createErrore:
descrizione_errore = createErrore.pgerror
codice_errore = createErrore.pgcode
return descrizione_errore, codice_errore
#pass
def updata_tabella(self):
pass
# for chiave, valore in val_prec.items():
# inserimento = "INSERT INTO " + self.schema + "." + self.nome_tabella + " (month, weight) VALUES (" + str(chiave) + "," + str(valore) + ");"
# self.cur.execute(inserimento)
def leggi_tabella(self):
conn_locale = psycopg2.connect("dbname=sparc_old user=postgres")
cur_locale = conn_locale.cursor()
comando_leggi_table = "SELECT ogc_fid FROM " + self.schema + "." + self.tabella_cicloni + ";"
try:
cur_locale.execute(comando_leggi_table)
records = cur_locale.fetchall()
return records
except psycopg2.Error as delErrore:
errore_delete_tabella = delErrore.pgerror
return errore_delete_tabella
def salva_cambi(self):
try:
self.cur.close()
self.conn.commit()
self.conn.close()
return "Changes saved"
except:
return "Problem in saving"
class UtilitieSparc(ProjectFlood):
campo_nome_paese = "ADM0_NAME"
campo_iso_paese = "ADM0_CODE"
campo_nome_admin = "ADM2_NAME"
campo_iso_admin = "ADM2_CODE"
nome_paese = ""
cod_paese = ""
def __init__(self, area, schema, tabella_pesi, tabella_pop_stat,
tabella_cicloni):
self.dati_per_plot = {}
self.dati_per_prob = {}
self.geolocator = Nominatim()
self.geolocator_geonames = GeoNames(country_bias=self.paese,
username='******',
timeout=1)
self.outDriver = ogr.GetDriverByName("ESRI Shapefile")
self.outShp = "classes/geocodifica/shp/" + self.paese + ".shp"
self.area = area
self.schema = schema
self.tabella_pesi = tabella_pesi
self.tabella_pop_stat = tabella_pop_stat
self.tabella_cicloni = tabella_cicloni
try:
self.conn = psycopg2.connect("dbname=sparc user=postgres")
except Exception as e:
print e.message
self.cur = self.conn.cursor()
def lista_admin0(self):
numFeatures = self.layer.GetFeatureCount()
lista_stati = []
for featureNum in range(numFeatures):
feature = self.layer.GetFeature(featureNum)
nome_paese = feature.GetField(self.campo_nome_paese)
lista_stati.append(nome_paese)
seen = set()
seen_add = seen.add
lista_pulita = [
x for x in lista_stati if not (x in seen or seen_add(x))
]
lista_admin0 = sorted(lista_pulita)
return tuple(lista_admin0)
def lista_admin2(self, country):
country_capitalized = country.capitalize()
self.layer.SetAttributeFilter(self.campo_nome_paese + " = '" +
country_capitalized + "'")
listone = {}
lista_iso = []
lista_clean = []
lista_admin2 = []
for feature in self.layer:
cod_admin = feature.GetField(self.campo_iso_admin)
nome_zozzo = feature.GetField(self.campo_nome_admin)
unicode_zozzo = nome_zozzo.decode('utf-8')
nome_per_combo = unicodedata.normalize('NFKD', unicode_zozzo)
no_dash = re.sub('-', '_', nome_zozzo)
no_space = re.sub(' ', '', no_dash)
no_slash = re.sub('/', '_', no_space)
no_apice = re.sub('\'', '', no_slash)
no_bad_char = re.sub(r'-/\([^)]*\)', '', no_apice)
unicode_pulito = no_bad_char.decode('utf-8')
nome_pulito = unicodedata.normalize('NFKD', unicode_pulito).encode(
'ascii', 'ignore')
lista_iso.append(cod_admin)
lista_clean.append(nome_pulito)
lista_admin2.append(nome_per_combo)
for i in range(len(lista_iso)):
listone[lista_iso[i]] = {
'name_orig': lista_admin2[i],
'name_clean': lista_clean[i]
}
return lista_admin2, listone
def creazione_struttura(self, admin_global):
# Check in data structures exists and in case not create the directory named
# after the country and all the directories
UtilitieSparc.proj_dir
os.chdir(UtilitieSparc.proj_dir)
country_low = str(self.paese).lower()
if os.path.exists(country_low):
os.chdir(UtilitieSparc.proj_dir + country_low)
admin_low = str(self.admin).lower()
if os.path.exists(admin_low):
pass
else:
os.mkdir(admin_low)
os.chdir(UtilitieSparc.proj_dir + country_low + "/" +
admin_global + "/")
os.mkdir("out")
else:
os.chdir(UtilitieSparc.proj_dir)
os.mkdir(country_low)
os.chdir(UtilitieSparc.proj_dir + country_low)
admin_low = str(self.admin).lower()
if os.path.exists(admin_low):
pass
else:
os.mkdir(admin_low)
os.chdir(UtilitieSparc.proj_dir + country_low + "/" +
admin_global + "/")
os.mkdir("out")
return "Project created......\n"
def create_template(self, dir_template):
os.chdir(dir_template)
tabella = dbf.Table('monthly_values', 'month C(10);mean N(5,1)')
return tabella.filename
def geolocate_accidents(self):
accidents = {}
with open(self.historical_table, 'rb') as csvfile:
luoghi_splittati = csv.reader(csvfile,
delimiter=",",
quotechar='"')
for row in luoghi_splittati:
if row[2] == self.paese:
id_incidente = str(row[9])
accidents[id_incidente] = {}
accidents[id_incidente]['paese'] = str(row[2])
accidents[id_incidente]['killed'] = str(row[6])
accidents[id_incidente]['affected'] = str(row[7])
accidents[id_incidente]['locations'] = {}
gruppo = str(row[3]).split(",")
quante_locations = len(gruppo)
for i in range(0, quante_locations):
accidents[id_incidente]['locations'][i] = gruppo[
i].strip()
totali = 0
successo = 0
insuccesso = 0
geocoding_success_file = "classes/geocodifica/text/" + self.paese + ".txt"
geocoding_fail_file = "classes/geocodifica/text/" + self.paese + "_fail.txt"
# Control if accidents have been geocoded already
if os.path.exists(geocoding_success_file):
print "Geocoded already!!"
pass
else:
geocoding_success = open(geocoding_success_file, "wb+")
geocoding_fail = open(geocoding_fail_file, "wb+")
geocoding_success.write("id,lat,lon\n")
geocoding_fail.write("id,lat,lon\n")
try:
for incidente in accidents.iteritems():
for location_non_geocoded in incidente[1][
'locations'].iteritems():
totali += 1
posto_attivo = location_non_geocoded[1]
if posto_attivo != 'NoData':
try:
print("Geocoding " + posto_attivo)
#location_geocoded = self.geolocator.geocode(posto_attivo, timeout=30)
location_geocoded = self.geolocator_geonames.geocode(
posto_attivo, timeout=30)
if location_geocoded:
scrittura = posto_attivo + "," + str(
location_geocoded.longitude
) + "," + str(
location_geocoded.latitude) + "\n"
geocoding_success.write(scrittura)
successo += 1
else:
geocoding_fail.write(posto_attivo + "," +
str(0) + "," +
str(0) + "\n")
insuccesso += 1
except ValueError as e:
print e.message
print "Total of %s events with %s successful %s unsuccessful and %d NULL" % (
str(totali), str(successo), str(insuccesso),
(totali - successo - insuccesso))
perc = float(successo) / float(totali) * 100.0
print "Percentage %.2f of success" % perc
except:
print "No response from geocoding server"
pass
def create_validated_coords(self):
def calc_poligono_controllo():
poligono = sf.bbox
global poligono_controllo
poligono_controllo = ((poligono[2], poligono[1]), (poligono[2],
poligono[3]),
(poligono[0], poligono[3]), (poligono[0],
poligono[1]))
global n
n = len(poligono_controllo)
def punti_dentro_poligono_di_controllo(x, y):
inside = False
p1x, p1y = poligono_controllo[0]
for i in range(n + 1):
p2x, p2y = poligono_controllo[i % n]
if y > min(p1y, p2y):
if y <= max(p1y, p2y):
if x <= max(p1x, p2x):
if p1y != p2y:
xinters = (y - p1y) * (p2x - p1x) / (p2y -
p1y) + p1x
if p1x == p2x or x <= xinters:
inside = not inside
p1x, p1y = p2x, p2y
return inside
def extract_country_shp():
# Get the input Layer
inShapefile = "C:/data/input_data/gaul_2014_2008_2/gaul_wfp.shp"
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(inShapefile, 0)
inLayer = inDataSource.GetLayer()
inLayer.SetAttributeFilter("ADM0_NAME = '" + self.paese + "'")
# Create the output LayerS
outShapefile = "C:/data/input_data/countries/" + self.paese + ".shp"
outDriver = ogr.GetDriverByName("ESRI Shapefile")
# Remove output shapefile if it already exists
if os.path.exists(outShapefile):
outDriver.DeleteDataSource(outShapefile)
# Create the output shapefile
outDataSource = outDriver.CreateDataSource(outShapefile)
out_lyr_name = os.path.splitext(os.path.split(outShapefile)[1])[0]
outLayer = outDataSource.CreateLayer(out_lyr_name,
geom_type=ogr.wkbMultiPolygon)
# Add input Layer Fields to the output Layer if it is the one we want
inLayerDefn = inLayer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
# Get the output Layer's Feature Definition
outLayerDefn = outLayer.GetLayerDefn()
# Add features to the ouput Layer
for inFeature in inLayer:
# Create output Feature
outFeature = ogr.Feature(outLayerDefn)
# Add field values from input Layer
for i in range(0, outLayerDefn.GetFieldCount()):
fieldDefn = outLayerDefn.GetFieldDefn(i)
outFeature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(),
inFeature.GetField(i))
# Set geometry as centroid
geom = inFeature.GetGeometryRef()
outFeature.SetGeometry(geom.Clone())
# Add new feature to output Layer
outLayer.CreateFeature(outFeature)
# Close DataSources
inDataSource.Destroy()
outDataSource.Destroy()
dentro = 0
fuori = 0
coords_check_file_in = "classes/geocodifica/text/" + self.paese + ".txt"
coords_validated_file_out = str('classes/geocodifica/csv/' +
str(self.paese) + '.csv')
if os.path.exists("C:/data/input_data/countries/" + self.paese +
".shp"):
sf = shapefile.Reader("C:/data/input_data/countries/" +
str(self.paese).lower() + ".shp")
calc_poligono_controllo()
else:
extract_country_shp()
sf = shapefile.Reader("C:/data/input_data/countries/" +
str(self.paese).lower() + ".shp")
calc_poligono_controllo()
with open(coords_check_file_in) as csvfile_in:
lettore_comma = csv.reader(csvfile_in,
delimiter=",",
quotechar='"')
next(lettore_comma)
with open(coords_validated_file_out, 'wb') as csvfile_out:
scrittore = csv.writer(csvfile_out,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
intestazioni = "id", "lat", "lon"
scrittore.writerow(intestazioni)
for row in lettore_comma:
if (punti_dentro_poligono_di_controllo(
float(row[1]), float(row[2]))):
stringa = str(row[0]), str(row[1]), str(row[2])
scrittore.writerow(stringa)
dentro += 1
else:
fuori += 1
csvfile_out.close()
csvfile_in.close()
#print "dentro %d" % dentro, "fuori %d" % fuori
def creazione_file_shp(self):
# Remove output shapefile if it already exists
if os.path.exists(self.outShp):
self.outDriver.DeleteDataSource(self.outShp)
#Set up blank lists for data
x, y, nomeloc = [], [], []
#read data from csv file and store in lists
with open('classes/geocodifica/csv/' + str(self.paese) + '.csv',
'rb') as csvfile:
r = csv.reader(csvfile, delimiter=';')
for i, row in enumerate(r):
if i > 0: #skip header
divisa = row[0].split(",")
#print divisa[0]
nomeloc.append(divisa[0])
x.append(float(divisa[1]))
y.append(float(divisa[2]))
#date.append(''.join(row[1].split('-')))#formats the date correctly
#target.append(row[2])
#Set up shapefile writer and create empty fields
w = shapefile.Writer(shapefile.POINT)
w.autoBalance = 1 #ensures gemoetry and attributes match
w.field('ID', 'N')
w.field('location', 'C', 50)
# w.field('Date','D')
# w.field('Target','C',50)
# w.field('ID','N')
#loop through the data and write the shapefile
for j, k in enumerate(x):
w.point(k, y[j]) #write the geometry
w.record(k, nomeloc[j]) #write the attributes
#Save shapefile
w.save(self.outShp)
def cancella_tabella(self):
comando_delete_table = "DROP TABLE " + self.schema + "." + self.tabella_pesi + " CASCADE;"
try:
self.cur.execute(comando_delete_table)
return "Table deleted"
except psycopg2.Error as delErrore:
errore_delete_tabella = delErrore.pgerror
return errore_delete_tabella
def crea_tabella(self):
try:
comando = "CREATE TABLE " + self.schema + "." + self.tabella_pesi + " (id serial PRIMARY KEY,month integer,weight double precision);"
print comando
self.cur.execute(comando)
#comando = "CREATE TABLE " + self.schema + "." + self.tabella_pesi + " (id serial PRIMARY KEY,month integer,weight double precision);"
#self.cur.execute(comando)
return "Table created"
except psycopg2.Error as createErrore:
descrizione_errore = createErrore.pgerror
codice_errore = createErrore.pgcode
return descrizione_errore, codice_errore
#pass
def updata_tabella(self):
pass
# for chiave, valore in val_prec.items():
# inserimento = "INSERT INTO " + self.schema + "." + self.nome_tabella + " (month, weight) VALUES (" + str(chiave) + "," + str(valore) + ");"
# self.cur.execute(inserimento)
def leggi_tabella(self):
conn_locale = psycopg2.connect("dbname=sparc_old user=postgres")
cur_locale = conn_locale.cursor()
comando_leggi_table = "SELECT ogc_fid FROM " + self.schema + "." + self.tabella_cicloni + ";"
try:
cur_locale.execute(comando_leggi_table)
records = cur_locale.fetchall()
return records
except psycopg2.Error as delErrore:
errore_delete_tabella = delErrore.pgerror
return errore_delete_tabella
def salva_cambi(self):
try:
self.cur.close()
self.conn.commit()
self.conn.close()
return "Changes saved"
except:
return "Problem in saving"
import pandas
# schoolguns.csv downloaded from http://www.gunviolencearchive.org/query
starterdf = pandas.read_csv("./schoolguns.csv")
cities = starterdf["City Or County"].tolist()
states = starterdf["State"].tolist()
killed = starterdf["# Killed"].tolist()
injured = starterdf["# Injured"].tolist()
Latitude = []
Longitude = []
geolocator = GeoNames(
username='******'
) # Register at Geonames.org . then activate free webservices at geonames.org/manageaccount
for city, state in zip(cities, states):
location = geolocator.geocode(city + "," + state, timeout=20)
if location is not None:
Latitude.append(location.latitude)
Longitude.append(location.longitude)
print(city + " " + state + " " + str(location.latitude) + " " +
str(location.longitude))
else:
Latitude.append(" ")
Longitude.append(" ")
df = pandas.DataFrame(
data={
"State": states,
"City or Count": cities,
"Latitude": Latitude,
"Longitude": Longitude,
feature_list.append(feature)
features['features'] = feature_list
with open("locations_full2.geojson", 'w+') as outfile:
json.dump(features, outfile)
#Main code
geolocator = GeoNames(username="******")
#replace with name of location file, or add in file input as arg
f = open("Locations4")
fail_list = list() # for recording a list of any place names that do not return a result
location_dict = dict()
#output_list = list()
for line in f:
unicode_line = unicode(line, "utf-8")
unicode_line = unicode_line [3:-3]
try:
location=geolocator.geocode(unicode_line )
coords = ((location.latitude, location.longitude))
location_dict[line] = coords
print unicode_line
print coords
except:
fail_list.append(unicode_line)
print 'FAILED:'
for failed in fail_list:
print failed
output()
class GeocodingEmDat(object):
def __init__(self, paese):
self.paese = paese
self.historical_table = "c:/data/tools/sparc/input_data/historical_data/floods - refine.csv"
self.geolocator = Nominatim()
self.geolocator_geonames = GeoNames(country_bias=self.paese,
username='******',
timeout=1)
self.outDriver = ogr.GetDriverByName("ESRI Shapefile")
self.countries_shp_location = os.getcwd() + '/input_data/countries'
self.outShp = os.getcwd(
) + "/input_data/geocoded/shp/" + self.paese + ".shp"
self.events_location = os.getcwd() + '/input_data/geocoded/shp/'
self.risk_map_location = os.getcwd() + '/input_data/geocoded/risk_map/'
def geolocate_accidents(self):
accidents = {}
with open(self.historical_table, 'rb') as csvfile:
luoghi_splittati = csv.reader(csvfile,
delimiter=",",
quotechar='"')
for row in luoghi_splittati:
if row[2] == self.paese:
id_incidente = str(row[9])
accidents[id_incidente] = {}
accidents[id_incidente]['paese'] = str(row[2])
accidents[id_incidente]['killed'] = str(row[6])
accidents[id_incidente]['affected'] = str(row[7])
accidents[id_incidente]['locations'] = {}
gruppo = str(row[3]).split(",")
quante_locations = len(gruppo)
for i in range(0, quante_locations):
accidents[id_incidente]['locations'][i] = gruppo[
i].strip()
totali = 0
successo = 0
insuccesso = 0
geocoding_success_file = "C:/data/tools/sparc/input_data/geocoded/text/" + self.paese + ".txt"
geocoding_fail_file = "C:/data/tools/sparc/input_data/geocoded/text/" + self.paese + "_fail.txt"
# Control if accidents have been geocoded already
if os.path.exists(geocoding_success_file):
return "Geocoded already!!"
pass
else:
geocoding_success = open(geocoding_success_file, "wb+")
geocoding_fail = open(geocoding_fail_file, "wb+")
geocoding_success.write("id,lat,lon\n")
geocoding_fail.write("id,lat,lon\n")
try:
for incidente in accidents.iteritems():
for location_non_geocoded in incidente[1][
'locations'].iteritems():
totali += 1
posto_attivo = location_non_geocoded[1]
if posto_attivo != 'NoData':
try:
print("Geocoding " + posto_attivo)
#location_geocoded = self.geolocator.geocode(posto_attivo, timeout=30)
location_geocoded = self.geolocator_geonames.geocode(
posto_attivo, timeout=30)
if location_geocoded:
scrittura = posto_attivo + "," + str(
location_geocoded.longitude
) + "," + str(
location_geocoded.latitude) + "\n"
geocoding_success.write(scrittura)
successo += 1
else:
geocoding_fail.write(posto_attivo + "," +
str(0) + "," +
str(0) + "\n")
insuccesso += 1
except ValueError as e:
print e.message
print "Total of %s events with %s successful %s unsuccessful and %d NULL" % (
str(totali), str(successo), str(insuccesso),
(totali - successo - insuccesso))
perc = float(successo) / float(totali) * 100.0
print "Percentage %.2f of success" % perc
except:
print "No response from geocoding server"
pass
def create_validated_coords(self):
def calc_poligono_controllo():
poligono = sf.bbox
global poligono_controllo
poligono_controllo = ((poligono[2], poligono[1]), (poligono[2],
poligono[3]),
(poligono[0], poligono[3]), (poligono[0],
poligono[1]))
global n
n = len(poligono_controllo)
def punti_dentro_poligono_di_controllo(x, y):
inside = False
p1x, p1y = poligono_controllo[0]
for i in range(n + 1):
p2x, p2y = poligono_controllo[i % n]
if y > min(p1y, p2y):
if y <= max(p1y, p2y):
if x <= max(p1x, p2x):
if p1y != p2y:
xinters = (y - p1y) * (p2x - p1x) / (p2y -
p1y) + p1x
if p1x == p2x or x <= xinters:
inside = not inside
p1x, p1y = p2x, p2y
return inside
def extract_country_shp():
# Get the input Layer
inShapefile = "input_data/gaul/gaul_wfp.shp"
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(inShapefile, 0)
inLayer = inDataSource.GetLayer()
inLayer.SetAttributeFilter("ADM0_NAME = '" + self.paese + "'")
# Create the output LayerS
outShapefile = "input_data/countries/" + self.paese + ".shp"
outDriver = ogr.GetDriverByName("ESRI Shapefile")
# Remove output shapefile if it already exists
if os.path.exists(outShapefile):
outDriver.DeleteDataSource(outShapefile)
# Create the output shapefile
outDataSource = outDriver.CreateDataSource(outShapefile)
out_lyr_name = os.path.splitext(os.path.split(outShapefile)[1])[0]
outLayer = outDataSource.CreateLayer(out_lyr_name,
geom_type=ogr.wkbMultiPolygon)
# Add input Layer Fields to the output Layer if it is the one we want
inLayerDefn = inLayer.GetLayerDefn()
for i in range(0, inLayerDefn.GetFieldCount()):
fieldDefn = inLayerDefn.GetFieldDefn(i)
outLayer.CreateField(fieldDefn)
# Get the output Layer's Feature Definition
outLayerDefn = outLayer.GetLayerDefn()
# Add features to the ouput Layer
for inFeature in inLayer:
# Create output Feature
outFeature = ogr.Feature(outLayerDefn)
# Add field values from input Layer
for i in range(0, outLayerDefn.GetFieldCount()):
fieldDefn = outLayerDefn.GetFieldDefn(i)
outFeature.SetField(
outLayerDefn.GetFieldDefn(i).GetNameRef(),
inFeature.GetField(i))
# Set geometry as centroid
geom = inFeature.GetGeometryRef()
outFeature.SetGeometry(geom.Clone())
# Add new feature to output Layer
outLayer.CreateFeature(outFeature)
# Close DataSources
inDataSource.Destroy()
outDataSource.Destroy()
dentro = 0
fuori = 0
coords_check_file_in = "input_data/geocoded/text/" + self.paese + ".txt"
coords_validated_file_out = str('input_data/geocoded/csv/' +
str(self.paese) + '.csv')
if os.path.exists("input_data/countries/" + self.paese + ".shp"):
sf = shapefile.Reader("input_data/countries/" +
str(self.paese).lower() + ".shp")
calc_poligono_controllo()
else:
extract_country_shp()
sf = shapefile.Reader("input_data/countries/" +
str(self.paese).lower() + ".shp")
calc_poligono_controllo()
with open(coords_check_file_in) as csvfile_in:
lettore_comma = csv.reader(csvfile_in,
delimiter=",",
quotechar='"')
next(lettore_comma)
with open(coords_validated_file_out, 'wb') as csvfile_out:
scrittore = csv.writer(csvfile_out,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
intestazioni = "id", "lat", "lon"
scrittore.writerow(intestazioni)
for row in lettore_comma:
if (punti_dentro_poligono_di_controllo(
float(row[1]), float(row[2]))):
stringa = str(row[0]), str(row[1]), str(row[2])
scrittore.writerow(stringa)
dentro += 1
else:
fuori += 1
csvfile_out.close()
csvfile_in.close()
#print "dentro %d" % dentro, "fuori %d" % fuori
def creazione_file_shp(self):
# Remove output shapefile if it already exists
if os.path.exists(self.outShp):
self.outDriver.DeleteDataSource(self.outShp)
#Set up blank lists for data
x, y, nomeloc = [], [], []
#read data from csv file and store in lists
with open('input_data/geocoded/csv/' + str(self.paese) + '.csv',
'rb') as csvfile:
r = csv.reader(csvfile, delimiter=';')
for i, row in enumerate(r):
if i > 0: #skip header
divisa = row[0].split(",")
#print divisa[0]
nomeloc.append(divisa[0])
x.append(float(divisa[1]))
y.append(float(divisa[2]))
#date.append(''.join(row[1].split('-')))#formats the date correctly
#target.append(row[2])
#Set up shapefile writer and create empty fields
w = shapefile.Writer(shapefile.POINT)
w.autoBalance = 1 #ensures gemoetry and attributes match
w.field('ID', 'N')
w.field('location', 'C', 50)
# w.field('Date','D')
# w.field('Target','C',50)
# w.field('ID','N')
#loop through the data and write the shapefile
for j, k in enumerate(x):
w.point(k, y[j]) #write the geometry
w.record(k, nomeloc[j]) #write the attributes
#Save shapefile
w.save(self.outShp)
def plot_mappa(self):
def GetExtent(gt, cols, rows):
ext = []
xarr = [0, cols]
yarr = [0, rows]
for px in xarr:
for py in yarr:
x = gt[0] + (px * gt[1]) + (py * gt[2])
y = gt[3] + (px * gt[4]) + (py * gt[5])
ext.append([x, y])
#print x,y
yarr.reverse()
return ext
pathToRaster = "input_data/geocoded/risk_map/" + self.paese + ".tif"
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import numpy as np
from osgeo import gdal
raster = gdal.Open(pathToRaster, gdal.GA_ReadOnly)
array = raster.GetRasterBand(1).ReadAsArray()
msk_array = np.ma.masked_equal(array, value=65535)
# print 'Raster Projection:\n', raster.GetProjection()
geotransform = raster.GetGeoTransform()
cols = raster.RasterXSize
rows = raster.RasterYSize
ext = GetExtent(geotransform, cols, rows)
#print ext[1][0], ext[1][1]
#print ext[3][0], ext[3][1]
#map = Basemap(projection='merc',llcrnrlat=-80, urcrnrlat=80, llcrnrlon=-180,urcrnrlon=180,lat_ts=20,resolution='c')
map = Basemap(projection='merc',
llcrnrlat=ext[1][1],
urcrnrlat=ext[3][1],
llcrnrlon=ext[1][0],
urcrnrlon=ext[3][0],
lat_ts=20,
resolution='c')
# Add some additional info to the map
map.drawcoastlines(linewidth=1.3, color='white')
#map.drawrivers(linewidth=.4, color='white')
map.drawcountries(linewidth=.75, color='white')
#datain = np.flipud(msk_array)
datain = np.flipud(msk_array)
map.imshow(
datain
) #,origin='lower',extent=[ext[1][0], ext[3][0],ext[1][1],ext[3][1]])
plt.show()
def add_prj(self):
env.workspace = self.events_location
inData = self.paese + ".shp"
print "Proietto " + inData
try:
coordinateSystem = "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]"
arcpy.DefineProjection_management(inData, coordinateSystem)
except arcpy.ExecuteError:
print arcpy.GetMessages(2)
arcpy.AddError(arcpy.GetMessages(2))
except Exception as e:
print e.args[0]
arcpy.AddError(e.args[0])
def create_heat_map(self):
# Local variables:
event_file_shp = self.events_location + self.paese + ".shp"
krn_map_file = self.risk_map_location + self.paese + ".tif"
try:
# Process: Kernel Density
arcpy.gp.KernelDensity_sa(event_file_shp, "NONE", krn_map_file,
"0.02", "", "SQUARE_MAP_UNITS")
except arcpy.ExecuteError:
print "Errore" + self.paese
print arcpy.GetMessages(2)
arcpy.AddError(arcpy.GetMessages(2))
except Exception as e:
print "Exception " + self.paese
print e.args[0]
arcpy.AddError(e.args[0])
def geocode(string): geolocator = GeoNames(username='******') return geolocator.geocode(string)
class GeoLocate(pygeoip.GeoIP):
"""
Geolocation class that inherits from GeoIP.
It also has an interface to geopy which
all in all seems a bit hacky to me.
You decide.
"""
def __init__(self, filename, geo_identity=None):
"""
Initializes the class.
Keyword arguments:
filename -- string representation of the file containing the geodata
geo_identity -- string representation of the identity in geonames
"""
pygeoip.GeoIP.__init__(self, filename)
self._setup_segments()
if geo_identity:
self.gnames = GeoNames(None, geo_identity)
def getOwnAddress(self):
"""Gets own address based on the IP Address of the user."""
self.address = str(self._getIPAddress())[2:-1]
(self.country, self.city, self.lat,
self.lon) = self.coordsFromAddr(*self._locateAddress())
def _getIPAddress(self):
"""
Gets own IP address using a web service. Is that too sloppy?
Returns:
string -- IP address
"""
return request.urlopen("http://bot.whatismyipaddress.com/").read()
def _locateAddress(self):
"""
Gets the city and country name for a certain location.
Returns:
Tuple of:
string -- city name
string -- country name
"""
return (self.record_by_addr(self.address)['city'],
self.record_by_addr(self.address)['country_name'])
def coordsFromAddr(self, cityname, countryname):
"""
Gets the coordinates for a certain city name.
Currently problematic with Korea; geonames
seems to have issues in that area.
Keyword Arguments:
cityname -- the name of the city searched for
countryname -- the name of the country searched for
Returns:
List of:
countryname -- country name provided
cityname -- city name provided
lat -- latitude of location (default None)
lon -- longitude of location (default None)
"""
try:
place, (lat, lon) = self.gnames.geocode(str(cityname) +
", " + str(countryname))
return [countryname, cityname, lat, lon]
except (TypeError, exc.GeopyError):
return [countryname, cityname, None, None]
def ownCoordsFromAddr(self, cityname, countryname):
"""
Gets the coordinates for the own city name
and makes the place found the own location.
Currently problematic with Korea; geonames
seems to have issues in that area.
Keyword Arguments:
cityname -- the name of the city searched for
countryname -- the name of the country searched for
Returns:
Boolean -- Indicates whether the coordinates could be parsed
"""
self.city = cityname
self.country = countryname
try:
place, (lat, lon) = self.gnames.geocode(str(cityname) +
", " + str(countryname))
self.lat = lat
self.lon = lon
return True
except (TypeError, exc.GeopyError):
self.lat = None
self.lon = None
return False
def lookForDup(self, location_list, location, treshold):
"""
Searches for points to be merged in a list of locations
within a certain treshold.
Keyword Arguments:
location_list -- list of known locations
location -- the location to be tested
treshold -- the treshold for the test
Returns:
Tuple of:
- Boolean -- Indicates whether the location could be merged
- ind -- Index of the location which fits (default None)
Known Exceptions:
Value Error:
Raised when gnames does not know the location or is not set
"""
if not self.gnames:
raise ValueError
try:
loc_tuple = Point(location[2], location[3])
d = distance.distance
ind = 0
for test in location_list:
test_tuple = Point(test[2], test[3])
if float(d(loc_tuple, test_tuple).miles) < treshold:
return False, ind
ind += 1
return True, None
except ValueError:
return True, None
