def geocode_all(db,
data_folder="geocoder/data",
terms_folder="geocoder/terms",
lines_per_insert=1000):
print("Loading table...")
# The query bellow seems not very efficient...
# Maybe change it as the link says.
# https://stackoverflow.com/questions/7389759/memory-efficient-built-in-sqlalchemy-iterator-generator
non_geocoded = Execucao.query.filter(Execucao.searched == False).all()
with Geocoder(data_folder, terms_folder) as geocoder:
counter = ProgressCounter(len(non_geocoded), print_abs=True)
to_be_inserted = 0
for row in non_geocoded:
cells = get_geolocable_cells(row)
geoent = geocoder.geocode_list(cells)
if geoent:
lat, lon, reg = geoent.best_coords()
if lat:
row.point = "POINT(%s %s)" % (lon, lat)
row.searched = True
to_be_inserted += 1
if to_be_inserted == lines_per_insert:
db.session.commit()
to_be_inserted = 0
counter.update()
if to_be_inserted:
db.session.commit()
counter.end()
def geocode_all(db, data_folder="geocoder/data",
terms_folder="geocoder/terms",
lines_per_insert=1000):
print("Loading table...")
non_geocoded = get_non_geocode(lines_per_insert)
if non_geocoded:
while non_geocoded:
with Geocoder(data_folder, terms_folder) as geocoder:
counter = ProgressCounter(len(non_geocoded), print_abs=True)
to_be_inserted = 0
for row in non_geocoded:
cells = get_geolocable_cells(row)
geoent = geocoder.geocode_list(cells)
if geoent:
lat, lon, reg = geoent.best_coords()
if lat:
row.point = "POINT(%s %s)" % (lon, lat)
row.searched = True
to_be_inserted += 1
if to_be_inserted == lines_per_insert:
db.session.commit()
to_be_inserted = 0
counter.update()
if to_be_inserted:
db.session.commit()
counter.end()
non_geocoded = get_non_geocode(lines_per_insert)
def get_coordinates(self):
geocoder = Geocoder()
uri = urlparse(self.path)
query = parse_qs(uri.query)
if query.get('address') is None:
return None
coordinates = geocoder.geocode(query['address'][0])
return coordinates
def _geocode(self, lat, lon):
geo = Geocoder()
df = geo.geocode_point((float(lat), float(lon)))
# import pdb; pdb.set_trace()
self.block_group = df[cn.BLOCK_GROUP].item()
self.neighborhood_long = df[cn.NBHD_LONG].item()
self.neighborhood_short = df[cn.NBHD_SHORT].item()
self.council_district = df[cn.COUNCIL_DISTRICT].item()
self.urban_village = df[cn.URBAN_VILLAGE].item()
self.zipcode = df[cn.ZIPCODE].item()
return self
def search(self,
keyword=None,
city=None,
state=None,
coordinates=None,
radius='5mi',
num=20):
auth = tweepy.OAuthHandler(k.consumer_key, k.consumer_secret)
auth.set_access_token(k.access_token, k.access_token_secret)
api = tweepy.API(auth)
geo = Geocoder()
print '\nGathering tweets...\n'
#kcs
if keyword is not None and city is not None and state is not None and coordinates is None:
coordinates = geo.getCoordinates(city, state)
coordinate_str = '{lat},{lng},'.format(**coordinates) + radius
tweets = api.search(q=keyword, geocode=coordinate_str, count=num)
#cs
if keyword is None and city is not None and state is not None and coordinates is None:
coordinates = geo.getCoordinates(city, state)
coordinate_str = '{lat},{lng},'.format(**coordinates) + radius
tweets = api.search(geocode=coordinate_str, count=num)
#kl
if keyword is not None and city is None and state is None and coordinates is not None:
coordinate_str = '{lat},{lng},'.format(**coordinates) + radius
tweets = api.search(q=keyword, geocode=coordinate_str, count=num)
#l
if keyword is None and city is None and state is None and coordinates is not None:
coordinate_str = '{lat},{lng},'.format(**coordinates) + radius
tweets = api.search(geocode=coordinate_str, count=num)
#k
if keyword is not None and city is None and state is None and coordinates is None:
tweets = api.search(q=keyword, count=num)
#st
tweets.sort(reverse=True, key=lambda x: x.author.followers_count)
print '\nAnalyzing data...\n'
a = SentimentAnalyzer()
return [{
'author': t.author.name,
"tweet": t.text,
"id": t.id,
"sentiment": a.analyze(t.text)
} for t in tweets]
def calculate_coords(self):
if self.city_name is None:
return
geocoder = Geocoder(self.city_name)
geocode = geocoder.get_geocode()
if geocode is None:
return
self.lat = geocode["lat"]
self.lon = geocode["lng"]
def main():
env = dotenv_values()
db_creds = {
key: env[key]
for key in ["dbname", "user", "password", "host", "port"]
}
conn = db.connect(**db_creds)
iter_conn = db.connect(**db_creds)
gc = Geocoder(conn, iter_conn, debug=False)
gc.geocode()
conn.close()
iter_conn.close()
def _handle_client(self, client, address):
PACKET_SIZE = 1024
while True:
print("CLIENT", client)
data = client.recv(PACKET_SIZE).decode()
if not data:
break
data_split = data.split(' ')
request_method = data_split[0]
if not request_method == "GET":
print("Unknown HTTP request method: {method}".format(
method=request_method))
else:
print("Successful method used: {method}".format(
method=request_method))
print("Method: {m}".format(m=request_method))
print("Request Body: {b}".format(b=data))
print("Data {data}".format(data=data_split))
http_code = 200
geocode_response = []
try:
geocoder = Geocoder(query_string_data=data_split[1])
geocode_response = geocoder.request()
if 'error' in geocode_response[0]:
http_code = 403
except Exception as e:
print("Geocoder request failed: {e}".format(e=e))
geocode_response = [{
"error": "error_initializing_geocoders"
}]
http_code = 500
response_header = self._generate_headers(http_code)
response = response_header.encode()
response += json.dumps(geocode_response)
client.send(response)
client.close()
break
def geocode_city(self, city_state_country):
client = Geocoder("http://open.mapquestapi.com/nominatim/v1/search?format=json")
response = client.geocode(city_state_country)
if (not response):
print ("no response for city=%s" % (city_state_country))
return None
#print (response)
#print response[0][self.DISPLAY_NAME]
#print response[0][self.BOUNDINGBOX]
#print response[0][self.ADDRESS]
'''return boundary of city in format [bottom, top, left, right]'''
return response[0][self.BOUNDINGBOX]
def ip(location, proxies='', timeout=5.0):
"""
Geocodes an IP address using MaxMind's services.
>>> g = geocoder.ip('74.125.226.99')
>>> g.latlng
(37.4192, -122.0574)
>>> g.address
'Mountain View, California United States'
...
Official Docs
-------------
http://www.maxmind.com/en/geolocation_landing
"""
provider = Ip(location)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def mapquest(location, proxies='', timeout=5.0):
"""
Retrieves geocoding data from MapQuest's address geocoding API.
>>> g = geocoder.mapquest('1555 Blake street, Denver')
>>> g.latlng
(39.740009, -104.992264)
>>> g.quality
'CITY'
...
Official Docs
-------------
http://www.mapquestapi.com/geocoding/
"""
provider = Mapquest(location)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def osm(location, proxies='', timeout=5.0):
"""
Retrieves geocoding data from OSM's data using Nominatim's geocoding API.
>>> g = geocoder.osm('Tacloban City')
>>> g.latlng
(11.2430274, 125.0081402)
>>> g.country
'Philippines'
...
Official Docs
-------------
http://wiki.openstreetmap.org/wiki/Nominatim
"""
provider = Osm(location)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def arcgis(location, proxies='', timeout=5.0):
"""
Retrieves geocoding data from ArcGIS's REST geocoding API.
>>> g = geocoder.arcgis('380 New York St, Redlands, California')
>>> g.latlng
(34.05649072776595, -117.19566584280369)
>>> g.postal
'92373'
...
Official Docs
-------------
http://resources.arcgis.com/en/help/arcgis-rest-api/
"""
provider = Arcgis(location)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def bing(location, key='', proxies='', timeout=5.0):
"""
Retrieves geocoding data from Bing's REST location API.
>>> key = 'XXXXX'
>>> g = geocoder.bing('Medina, Washington', key=key)
>>> g.latlng
(47.615821838378906, -122.23892211914062)
>>> g.country
'United States'
...
Official Docs
-------------
http://msdn.microsoft.com/en-us/library/ff701714.aspx
"""
provider = Bing(location, key=key)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def tomtom(location, key='', proxies='', timeout=5.0):
"""
Retrieves geocoding data from TomTom's geocoding API.
>>> key = 'XXXXX'
>>> g = geocoder.tomtom('Amsterdam, Netherlands', key=key)
>>> g.latlng
(52.373166, 4.89066)
>>> g.quality
'city'
...
Official Docs
-------------
http://developer.tomtom.com/products/geocoding_api
"""
provider = Tomtom(location, key=key)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def _get_blockgroup(self, df):
geo = Geocoder()
temp = []
for chunk in self._chunker(df, 250):
coords = chunk.loc[:, (cn.LAT, cn.LON)]
blkgrps = geo.geocode_df(coords)
blkgrps.columns = [
cn.LAT, cn.LON, cn.DEST_BLOCK_GROUP, cn.NBHD_LONG,
cn.NBHD_SHORT, cn.COUNCIL_DISTRICT, cn.URBAN_VILLAGE,
cn.ZIPCODE
]
temp.append(
pd.merge(chunk,
blkgrps,
left_on=[cn.LAT, cn.LON],
right_on=[cn.LAT, cn.LON],
how='left').drop_duplicates())
df = pd.concat(temp, sort=False).reset_index()
return df
def reverse(latlng, proxies='', timeout=5.0):
"""
Reverse geocodes a location based on Lat & Lng inputs
using Google's reverse geocoding API V3.
>>> latlng = (37.4192, -122.0574)
>>> g = geocoder.reverse(latlng)
>>> g.address
'Sevryns Road, Mountain View, CA 94043, USA'
>>> g.postal
'94043'
...
Official Docs
-------------
https://developers.google.com/maps/documentation/geocoding/
"""
provider = Reverse(latlng)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def nokia(location, app_id='', app_code='', proxies='', timeout=5.0):
"""
Retrieves geocoding data from Nokia's HERE geocoder API.
>>> app_id = 'XXXXX'
>>> app_code = 'XXXXX'
>>> g = geocoder.nokia('Keilaniemi, Espoo')
>>> g.latlng
(60.1759338, 24.8327808)
>>> g.country
'FIN'
...
Official Docs
-------------
https://developer.here.com/rest-apis/documentation/geocoder
"""
provider = Nokia(location, app_id=app_id, app_code=app_code)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def geonames(location, username='', proxies='', timeout=5.0):
"""
Retrieves geocoding data from Geonames's Web Service API.
>>> username = '******'
>>> g = geocoder.geonames('Springfield, Virginia', username=username)
>>> g.latlng
(38.78928, -77.1872)
>>> g.country
'United States'
>>> g.population
30484
...
Official Docs
-------------
http://www.geonames.org/export/web-services.html
"""
provider = Geonames(location, username=username)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def __init__(self, origin_lat: float, origin_lon: float,
mapquest_api_key: str) -> None:
"""
Creates a new monitor.
Args:
origin_lat (float): The origin latitude.
origin_lon (float): The origin longitude.
mapquest_api_key (str): The MapQuest API key for geocoding.
"""
self._ses = requests.Session()
self._parser = RealTimeParser()
self._coder = Geocoder(mapquest_api_key)
self._origin = (origin_lat, origin_lon)
self._incidents = {} # type: Dict[str, Incident]
self._ses.headers.update({
"User-Agent":
"SFD Feed Watcher "
"(https://github.com/xyx0826/sea_fires_around_me)"
})
def check_input(self, location):
lat, lng = 0.0, 0.0
# Checking for a String
if isinstance(location, str):
lat, lng = Geocoder(Google(location)).latlng
# Checking for List of Tuple
if isinstance(location, (list, tuple)):
lat, lng = self.check_for_list(location)
# Checking for Dictionary
elif isinstance(location, dict):
lat, lng = self.check_for_dict(location)
# Checking for a Geocoder Class
elif hasattr(location, 'latlng'):
lat, lng = location.latlng
# Return Results
return lat, lng
def google(location,
client='',
secret='',
proxies='',
api_key='',
timeout=5.0):
"""
Retrieves geocoding data from Google's geocoding API V3
>>> g = geocoder.google('1600 Amphitheatre Pkwy, Mountain View, CA')
>>> g.latlng
(37.784173, -122.401557)
>>> g.country
'United States'
...
Official Docs
-------------
https://developers.google.com/maps/documentation/geocoding/
"""
provider = Google(location, client=client, secret=secret, api_key=api_key)
return Geocoder(provider, proxies=proxies, timeout=timeout)
def __new_bot(self):
timezone = TimezoneApi('bar')
timezone.load = MagicMock(return_value=12345)
darksky = WeatherSource('foo', timezone)
darksky.load = MagicMock(
return_value=Weather(now=WeatherAtTime(20, 'Clear', ''),
day=WeatherDay(19, 21, 'Mostly Cloudy', '')))
geocoder = Geocoder('foo')
geocoder.geocode = MagicMock(return_value={
'results': [{
'geometry': {
'location': {
'lat': 1.2,
'lng': 3.4
}
}
}]
})
webcam_source = WebcamSource('foo')
webcam_source.load = MagicMock(return_value=None)
return WeatherBot(darksky, geocoder, webcam_source)
from geocoder import Geocoder
from time import sleep
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException
geo = Geocoder()
driver = geo.set_driver(show=True, image=True)
address = "청와대"
driver.get("https://www.google.com/maps/place/{}".format(address))
driver.implicitly_wait(3)
driver.find_element_by_xpath("//*[@id='searchbox-searchbutton']").click()
driver.implicitly_wait(3)
try:
addr_converted = driver.find_element_by_class_name(
"section-result-details-container").text.strip()
except NoSuchElementException:
addr_converted = driver.find_element_by_xpath(
"//*[@id='pane']/div/div[1]/div/div/div[8]/div/div[1]/span[3]/span[3]"
).text.strip()
driver.find_element_by_xpath("//*[@id='searchboxinput']").clear()
driver.find_element_by_xpath("//*[@id='searchboxinput']").send_keys(
addr_converted)
driver.find_element_by_xpath("//*[@id='searchbox-searchbutton']").click()
sleep(0.5)
pos1 = {"xoffset": round(1920 / 2 - 115), "yoffset": round(1080 / 2 - 120)}
loc_search_actions = ActionChains(driver)
loc_search_actions.move_by_offset(**pos1)
def set_providers_to_geocoder():
providers = Provider.objects.order_by('order')
global geocoder
geocoder = Geocoder()
for provider in providers:
geocoder.add_provider(provider)
# reflect an existing database into a new model
Base = automap_base()
# reflect the tables
Base.prepare(engine, reflect=True)
# Save references to each table
address_data = Base.classes.Address_Data
census_data = Base.classes.Census_Data
# Create the locations dataframe with all data, including both statuses
loc_df = pd.read_sql_table("Address_Data", con=engine)
# Create a locations dataframe with reference/seeded/locked locations
ref_loc_df = loc_df[loc_df.Status == "S"]
ref_loc_df = ref_loc_df.loc[:, ["Latitude", "Longitude", "StreetAddress"]]
# Call the geocoder class
geo = Geocoder(ref_loc_df)
#################################################
# Flask Routes
#################################################
# Route to render index.html template using data from db
@app.route("/")
def index():
"""Returns all street names to populate street name dropdown in HTML for selection"""
# select only StreetAddress and get rid of the number and return only unique values
select_st = 'select distinct right("StreetAddress", (length("StreetAddress") - position(\' \' in "StreetAddress"))) "StreetAddress"\
from "Address_Data" a'
import config
import prompts
import requests
from db_handler import DB
from geocoder import Geocoder
from datetime import timedelta
from flask import Flask, request, session, render_template, url_for
from twilio.twiml.messaging_response import Body, Media, Message, MessagingResponse
db = DB()
app = Flask(__name__)
app.secret_key = config.secret_key
app.config.from_object(__name__)
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(minutes=5)
geocoder = Geocoder(config.api_key)
@app.route('/', methods=['GET'])
def root():
db.execute(config.db_credentials, config.select_all_records)
data = db.get_all_records()
return render_template('index.html', data=data)
@app.route('/map', methods=['GET'])
def map():
db.execute(config.db_credentials, config.select_all_records)
data = db.get_all_records()
return render_template('map.html', data=data)
def text():
"""Respond to incoming texts with a message."""
resp = MessagingResponse()
# get sender phone number, check if they are a current subscriber
incoming_number = request.values.get('From')[2:]
if incoming_number in users.keys():
caller = users[incoming_number]
else:
caller = contact.Contact(incoming_number)
# get body of incoming SMS and format it
body = request.values.get('Body')
b = body.upper().strip()
# check if the body is 'HEALTH', 'EDU', 'ADD', 'REMOVE' or anything else
if b in keywords['Health']:
health_msg = message.HealthMsg().make_msg()
resp.message(health_msg)
print("{} texted HEALTH".format(incoming_number))
myLogger.emit(logging.INFO, "HEALTH texted", request.values)
elif b in keywords['Edu']:
call_msg = message.CallMsg().make_msg()
resp.message(call_msg)
print("{} requested a call from a Health Educator".format(
incoming_number))
myLogger.emit(logging.INFO, "Health Educator call requested",
request.values)
# get users last requested address or the address they're subscribed to
if incoming_number in users.keys():
edu_addr = caller.last_requested_address
else:
edu_addr = [a[0] for a in caller.addresses][0]
# calculate demos nearby to generate short report for Health Dept
located = Geocoder().geocode(edu_addr)
# query Socrata datasets
scheduled_demos = soda_client.get(
"tsqq-qtet",
where="within_circle(location, {}, {}, 155)".format(
located['location']['y'], located['location']['x']))
pipeline_demos = soda_client.get(
"dyp9-69zf",
where="within_circle(location, {}, {}, 155)".format(
located['location']['y'], located['location']['x']))
past_demos = soda_client.get(
"rv44-e9di",
where="within_circle(location, {}, {}, 155)".format(
located['location']['y'], located['location']['x']))
upcoming_demos = scheduled_demos + pipeline_demos
# format 'em
demo_dates = []
if len(scheduled_demos) > 0:
for d in scheduled_demos:
demo_date = "{}".format(
datetime.strptime(
(d['demolish_by_date']),
'%Y-%m-%dT%H:%M:%S').strftime('%m-%d-%Y'))
demo_dates.append(demo_date)
else:
demo_date = None
demo_dates.append(demo_date)
# send log to Loggly
log_extras = {
"last_address_texted": edu_addr,
"upcoming_demos_count": len(upcoming_demos),
"past_demos_count": len(past_demos),
"next_knockdown_date": demo_dates[0]
}
myLogger.emit(logging.INFO,
"Health Educator call requested.",
request.values,
last_address_texted=edu_addr,
upcoming_demo_count=len(upcoming_demos),
next_knockdown_date=demo_dates[0])
# send request to Slack
webhook_url = os.environ['SLACK_WEBHOOK_URL']
caller_msg = ":phone: `{}` requested a call from a Health Educator \nLast address texted: *{}* \nNumber of upcoming demos: *{}* \nNumber of past demos: *{}* \nNext knock-down date: *{}*".format(
incoming_number, edu_addr, len(upcoming_demos), len(past_demos),
demo_dates[0])
slack_data = {'text': caller_msg}
response = requests.post(webhook_url,
data=json.dumps(slack_data),
headers={'Content-Type': 'application/json'})
if response.status_code != 200:
myLogger.emit(logging.ERROR,
"Slack request failed", [],
status_code=response.status_code,
response=response.text)
raise ValueError(
'Request to slack returned an error %s, the response is:\n%s' %
(response.status_code, response.text))
elif b in keywords['Add'] and caller.last_requested_address:
caller.watch(caller.last_requested_address)
msg = message.SubscribeMsg(caller.last_requested_address)
success_msg = msg.make_msg()
resp.message(success_msg)
# remove from users so we grab a 'fresh' copy of the user with sheet rows
del users[incoming_number]
print("{} subscribed to {}".format(incoming_number,
caller.last_requested_address))
myLogger.emit(logging.INFO,
"Caller subscribed to an address",
request.values,
address=caller.last_requested_address)
elif b in keywords['Remove']:
for address in caller.addresses:
caller.unwatch(address)
msg = message.UnsubscribeMsg([a[0] for a in caller.addresses])
remove_msg = msg.make_msg()
resp.message(remove_msg)
print("{} unsubscribed from {} addresses".format(
incoming_number, len(caller.addresses)))
myLogger.emit(logging.INFO,
"Caller unsubscribed to a number of addresses",
request.values,
address_count=len(caller.addresses))
else:
pattern = re.compile('(\d{2,5})\s?(\w{2,})')
result = pattern.search(body)
if result:
send = result.group(1) + ' ' + result.group(2)
else:
send = body
# send it to the geocoder
located = Geocoder().geocode(send)
# if it's a valid address, build up a text message with demos nearby
if located:
print("Geocoded {} from {}".format(located['address'],
incoming_number))
myLogger.emit(logging.INFO,
"Geocoded an address",
request.values,
geocode=located["address"])
msg = message.DemoMsg(located)
demo_msg = msg.make_msg()
resp.message(demo_msg)
# store matched address
caller.last_requested_address = located['address'][:-7]
users[incoming_number] = caller
# default message for a bad address
else:
default_msg = message.DefaultMsg().make_msg()
resp.message(default_msg)
print("Couldn't geocode '{}' from {}; Sent it to Slack".format(
body, incoming_number))
myLogger.emit(logging.ERROR,
"Geocoding failed",
request.values,
from_address=body)
# send it to Slack
webhook_url = os.environ['SLACK_WEBHOOK_URL']
err_msg = ":exclamation: demo-alerts can't geocode `{}` from `{}`".format(
send, incoming_number)
slack_data = {'text': err_msg}
response = requests.post(
webhook_url,
data=json.dumps(slack_data),
headers={'Content-Type': 'application/json'})
if response.status_code != 200:
myLogger.emit(logging.ERROR,
"Slack request failed", [],
status_code=response.status_code,
response=response.text)
raise ValueError(
'Request to slack returned an error %s, the response is:\n%s'
% (response.status_code, response.text))
# send the text
return str(resp)
from geocoder import Geocoder
from query_parser import QueryParser
PORT = 9000
RESPONSES = [
"Hrm... How about asking me about the weather?",
"I don't know about that... Want to know about the weather?",
"I'm not really into that type of thing. Let's talk about the weather.",
"You must be off your rocker at the moment. Come back when you want to talk weather."
]
app = Flask(__name__)
CORS(app)
store = {}
geocoder = Geocoder()
@app.route('/chat/messages', methods=['GET', 'POST'])
def chat_messages():
if request.form['action'] == 'join':
store[request.form['user_id']] = {
"messages": ["Hello %s!" % (request.form['name'])]
}
reply = store[request.form['user_id']]['messages'][-1]
elif request.form['action'] == 'message':
location = QueryParser(request.form['text']).location()
if location:
lat, lng = geocoder.encode(location)
reply = Forecaster(lat, lng).weather_summary()
import timeit
from geocoder import Geocoder
g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')
print g.address_to_latlng('10 WEST 49 STREET')
print g.address_to_latlng('10 WEST 49 ST')
print g.address_to_latlng('10 W 49 ST')
print timeit.timeit("""g.address_to_latlng('99 BROADWAY')""",
setup="""from geocoder import Geocoder
g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')""")
'''
print timeit.timeit("""g.latlng_to_address((40.769083827890107,-73.952477804528087))""",setup="""from geocoder import Geocoder
g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')""")
'''
