def test_tolonlat(self):
x, y = stateplane.to_lonlat(817080.8169336573, 99364.28495057777, epsg='32129')
self.assertAlmostEqual(x, -75.2)
self.assertAlmostEqual(y, 40.2)
y, x = stateplane.to_latlon(817080.8169336573, 99364.28495057777, epsg='32129')
self.assertAlmostEqual(x, -75.2)
self.assertAlmostEqual(y, 40.2)
def test_tolonlat(self):
x, y = stateplane.to_lonlat(817080.8169336573, 99364.28495057777, epsg='32129')
self.assertAlmostEqual(x, -75.2)
self.assertAlmostEqual(y, 40.2)
y, x = stateplane.to_latlon(817080.8169336573, 99364.28495057777, epsg='32129')
self.assertAlmostEqual(x, -75.2)
self.assertAlmostEqual(y, 40.2)
def run(self, input_file):
with open(input_file) as fp:
for line in fp:
line = line.strip()
s = line.split(',')
x = float(s[6].strip())
y = float(s[7].strip())
x *= 0.3048
y *= 0.3048
lat, lon = stateplane.to_latlon(x, y, epsg='32129')
print("%s,%f,%f" % (line, lat, lon))
def get(self):
easting = self.get_argument('easting', None)
northing = self.get_argument('northing', None)
epsg = self.get_argument('epsg', None)
fips = self.get_argument('fips', None)
abbr = self.get_argument('abbr', None)
if easting is None or northing is None:
self.write('*You must given easting or northing.')
return
easting = float(easting) * (1200 / 3937)
northing = float(northing) * (1200 / 3937)
if epsg is not None:
print('easting={}, northing={}, epsg={}'.format(easting, northing, epsg))
self.write('{},{}'.format(*sp.to_latlon(easting, northing, epsg=epsg)))
elif fips is not None:
print('easting={}, northing={}, fips={}'.format(easting, northing, fips))
self.write('{},{}'.format(*sp.to_latlon(easting, northing, fips=fips)))
elif abbr is not None:
print('easting={}, northing={}, abbr={}'.format(easting, northing, abbr))
self.write('{},{}'.format(*sp.to_latlon(easting, northing, abbr=abbr)))
else:
self.write('*As least one of epsg, fips and abbr must be provided.')
def extract_intersection(east, north):
# This function takes two inputs: easting and northing of location in feet
# Finds the nearest street and the next crossing street for a given lat/lng pair.
# findNearestIntersectionOSM API
# http://www.geonames.org/maps/osm-reverse-geocoder.html
# Ex) extract_intersections(42.3516652, -071.1212753)
#
# 1 foot = 0.3048 meter
conv = 0.3048
(lat, lon) = stateplane.to_latlon(east*conv, north*conv, fips='2001')
base_url = "http://api.geonames.org/findNearestIntersectionOSMJSON?"
request = base_url + "lat=" + str(lat) + "&lng=" + str(lon) + "&username=andrewenze"
response = urllib.request.urlopen(request).read().decode("utf-8")
r = json.loads(response)
# s = json.dumps(r, sort_keys=True, indent=2)
str1 = r['intersection']
str2 = r['intersection']
return (str1['street1'], str2['street2'])
def extract_intersection(east, north):
# This function takes two inputs: easting and northing of location in feet
# Finds the nearest street and the next crossing street for a given lat/lng pair.
# findNearestIntersectionOSM API
# http://www.geonames.org/maps/osm-reverse-geocoder.html
# Ex) extract_intersections(42.3516652, -071.1212753)
#
# 1 foot = 0.3048 meter
conv = 0.3048
(lat, lon) = stateplane.to_latlon(east * conv, north * conv, fips="2001")
base_url = "http://api.geonames.org/findNearestIntersectionOSMJSON?"
request = base_url + "lat=" + str(lat) + "&lng=" + str(lon) + "&username=andrewenze"
response = urllib.request.urlopen(request).read().decode("utf-8")
r = json.loads(response)
# s = json.dumps(r, sort_keys=True, indent=2)
str1 = r["intersection"]
str2 = r["intersection"]
return (str1["street1"], str2["street2"])
df0 = pd.read_excel(os.path.join(tmpdir, f), sheetname=0)
df1 = pd.read_excel(os.path.join(tmpdir, f), sheetname=1)
frames = [df0, df1]
df = pd.concat(frames)
df = df.set_index('TSR ID')
os.remove(os.path.join(tmpdir, f))
os.rmdir(tmpdir)
# # set path and import
# convert stateplane coords to lat long
df[['geox', 'geoy']] = df[['geox', 'geoy']].apply(lambda x: sp.to_latlon(x['geox']/100*0.3048,
x['geoy']/100*0.3048, epsg='2264'), axis=1) # convert
# change column names and remove odd values
df.columns = [x.lower() for x in df.columns]
rx = re.compile('\W+') # get all nonnumeric values (including spaces)
df.columns = [rx.sub('_', v) for v in df.columns] # all spaces to underscores
# FIX errors in 'state'
df.state = 'NC'
# change all nonnumeric values to strings
## code from http://stackoverflow.com/questions/20670370/pandas-and-unicode
types = df.apply(lambda x: pd.lib.infer_dtype(x.values))
for col in types[(types=='unicode') | (types=='mixed')].index:
df[col] = df[col].astype(str)
ageColumn = i
if columns[i] == "datestop":
dateColumn = i
if columns[i] == "race":
raceColumn = i
if columns[i] == "arstmade":
arstColumn = i
if columns[i] == "ycoord":
latColumn = i
if columns[i] == "xcoord":
lonColumn = i
trimmedFile = open(trimmedPath + "/" + rawFileName, 'w')
for line in rawFile:
columns = line.split(",");
sex = columns[sexColumn]
age = columns[ageColumn]
date = columns[dateColumn]
race = columns[raceColumn]
arst = columns[arstColumn]
latString = columns[latColumn].strip()
lonString = columns[lonColumn].strip()
if len(latString) > 5 and len(lonString) > 5:
lat = float(latString) * .3048
lon = float(lonString) * .3048
latlon = stateplane.to_latlon(lon, lat, epsg="32118")
trimmedFile.write(sex+","+age+","+date+","+race+","+arst+","+str(latlon[0])+","+str(latlon[1])+"\n")
trimmedFile.close()
rawFile.close()
def _get_town_coords(self,
counties=False,
from_file='csv',
save_to_csv=False):
"""
Reads in and creates dictionary of mass town lat/lon and SPCS coords from file type specified with from_file,
and save to file at MASS_TOWNS_COORDS_PATH if save_to_csv is true. If counties is true the same process is
done except with mass counties instead of towns.
:param counties: (bool) indicates whether to generate coords for counties or towns
:param from_file: (str) file type abbreviation which decides how coords are generated
- 'csv', gets points from mass_towns_coords.csv or mass_counties_coords.csv
- 'shp', generates points with geopy from from centroids of mass towns shapefile's polygons
:param save_to_csv: (bool) indicates whether generated points will be saved to a csv file on disk
:return: coordinates dictionary of towns or counties label where each label maps to another dictionary with
keys 'lat','lon','x', and 'y'
"""
towns_df = None
# convert mass_towns_coords.csv to pandas dataframe
if from_file == 'csv':
path = MASS_TOWNS_COORDS_PATH if not counties else MASS_COUNTIES_COORDS_PATH
towns_df = pd.read_csv(path)
# generate points with geopy
elif from_file == 'shp':
names = []
centroid_x = []
centroid_y = []
centroid_lat = []
centroid_lon = []
# use geopy to get town coords from centroids of shapefile polygons
gdf = self.mass_towns_geoframe if not counties else self.mass_counties_geoframe
for row in gdf.itertuples():
x, y = row.center.x, row.center.y # SCPS coords
lat, lon = stateplane.to_latlon(x, y, abbr='MA_M')
location = self.geolocator.reverse(f"{lat}, {lon}")
if not counties:
# get name of town, city, or village
address_dict = location.raw['address']
if 'town' in address_dict:
town_name = address_dict['town']
elif 'city' in address_dict:
town_name = address_dict['city']
elif 'village' in address_dict:
town_name = address_dict['village']
else:
print(
'Unable to get town, city, or village name from location at centroid point '
f'from shapefile with info:\n{address_dict}')
else:
# get county name
town_name = location.raw['address']['county']
# some centroid points lay right on border and geopy will map them to a different town or city
# than what is in the TOWN or COUNTY column of self.mass_towns_geoframe or self.mass_counties_geoframe,
# in this case use the representative point instead of the centroid
true_name = row.TOWN if not counties else row.COUNTY
if not self._locations_related(true_name, town_name):
x, y = row.representative_point.x, row.representative_point.y # SCPS coords
lat, lon = stateplane.to_latlon(x, y, abbr='MA_M')
location = self.geolocator.reverse(f"{lat}, {lon}")
if not counties:
# get name of town, city, or village
address_dict = location.raw['address']
if 'town' in address_dict:
town_name = address_dict['town']
elif 'city' in address_dict:
town_name = address_dict['city']
elif 'village' in address_dict:
town_name = address_dict['village']
else:
print(
'Unable to get town, city, or village name from location at centroid point '
f'from shapefile with info:\n{address_dict}')
else:
# get county name
town_name = location.raw['address']['county']
if not self._locations_related(true_name, town_name):
print(
f'Geopy location label of "{town_name}" does not mathc coords of "{true_name}"'
)
# even if location names are still unrelated town_name is still set to the true_name since the location
# coords are still within the shapefile polygon for the location
town_name = string.capwords(true_name)
names.append(town_name)
centroid_x.append(x)
centroid_y.append(y)
centroid_lat.append(lat)
centroid_lon.append(lon)
# centroid_town and centroid_county should have the same values as TOWN and COUNTY in
# self.mass_towns_geoframe and self.mass_counties_geoframe
label_type = 'centroid_town' if not counties else 'centroid_county'
towns_df = pd.DataFrame({
label_type: names,
'centroid_x': centroid_x,
'centroid_y': centroid_y,
'centroid_lat': centroid_lat,
'centroid_lon': centroid_lon
})
if save_to_csv and towns_df is not None:
path = MASS_TOWNS_COORDS_PATH if not counties else MASS_COUNTIES_COORDS_PATH
towns_df.to_csv(path, index=False)
return towns_df
def test_tolonlat_err(self):
with self.assertRaises(ValueError):
stateplane.to_latlon(817080.8169336573, 99364.28495057777)
stateplane.to_lonlat(817080.8169336573, 99364.28495057777)
def test_tolonlat_err(self):
with self.assertRaises(ValueError):
stateplane.to_latlon(817080.8169336573, 99364.28495057777)
stateplane.to_lonlat(817080.8169336573, 99364.28495057777)
