def main(args_dict):
test_mode = not args_dict['disable_test_mode']
if test_mode:
print "======================="
print "=======TEST MODE======="
print "======================="
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
# Write output into prod_schema_name
prod_schema_name = "prod_" + timestamp
print "prod_schema_name", prod_schema_name
# Create database connections:
# Read / write address cache from this one
db_address_cache = DatabaseConnection(
path_config='db_config_update_source.yaml',
search_path='address_cache')
# Write prod tables into this one
db_prod = DatabaseConnection(path_config='db_config_update_source.yaml')
CreateAndSetProdSchema(db_prod, prod_schema_name)
# Initialize geocoder
geocoder = geocoder_lib.Geocoder(db_address_cache, db_prod, test_mode)
# Initialize entity lookup
entities_lookup = entities.Entities(db_prod)
# Table prod_tables.yaml defines a specifications of SQL selects to read
# source data and describtion of additional tables to be created.
with open('prod_tables.yaml', 'r') as stream:
config = yaml.load(stream)
# This is where all the population happens!!!
# Go through all the specified data sources and process them, adding data
# as needed. We process them in the order!
for key in sorted(config.keys()):
config_per_source = config[key]
print "Working on source:", key
ProcessSource(db_prod, geocoder, entities_lookup, config_per_source,
test_mode)
print "GEOCODER STATS"
geocoder.PrintStats()
# Grant apps read-only access to the newly created schema and tables within
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'data')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'verejne')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'kataster')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'prepojenia')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'obstaravania')
# Commit database changes and close database connections
db_address_cache.commit()
db_address_cache.close()
if test_mode:
db_prod.conn.rollback()
else:
db_prod.commit()
db_prod.close()
def geocode():
coder = geocoder.Geocoder()
lat = request.args.get('lat')
lon = request.args.get('lon')
address = request.args.get('address')
if not lat and not lon and not address:
return make_response(jsonify({'error' : 'No arguments provided'}), 500)
latlng = None
if lat and lon:
latlng = lat + "," + lon
result = coder.lookup(address, latlng)
return make_response(jsonify(result))
def post(self):
data = ListingForm(data=self.request.POST)
if data:
logging.info("Data: %s" % data)
logging.info(vars(data))
geoPoint = None
g = geocoder.Geocoder()
pts = g.query(self.request.POST.get('address'))
if pts and len(pts) >= 2:
geoPoint = db.GeoPt(pts[0], pts[1])
logging.info("Using lat %s and long %s" % (pts[0], pts[1]))
# Save the data, and redirect to the view page
entity = None
photo = self.request.get("photo")
if photo:
entity.photo = db.Blob(photo)
entity = data.save(commit=False)
entity.location = geoPoint
entity.update_location()
entity.author = users.get_current_user()
entity.put()
self.redirect('/?success=yes')
else:
# Reprint the form
self.response.out.write('<html><body>'
'<form method="POST" '
'action="/listing/add">'
'<table>')
self.response.out.write(data)
self.response.out.write('</table>'
'<input type="submit">'
'</form></body></html>')
def main(args_dict):
test_mode = not args_dict['disable_test_mode']
if test_mode:
print "======================="
print "=======TEST MODE======="
print "======================="
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
# Write output into prod_schema_name
prod_schema_name = "prod_" + timestamp
print "prod_schema_name", prod_schema_name
# Create database connections:
db_source = DatabaseConnection(path_config='db_config_update_source.yaml')
db_address_cache = DatabaseConnection(
path_config='db_config_update_source.yaml',
search_path='address_cache')
db_prod = DatabaseConnection(path_config='db_config_update_source.yaml')
CreateAndSetProdSchema(db_prod, prod_schema_name)
# Initialize geocoder
geocoder = geocoder_lib.Geocoder(db_address_cache, db_prod, test_mode)
# Initialize entity lookup
entities_lookup = entities.Entities(db_prod)
# Table prod_tables.yaml defines a specifications of SQL selects to read
# source data and describtion of additional tables to be created.
config = utils.yaml_load('prod_tables.yaml')
# This is where all the population happens!!!
# Go through all the specified data sources and process them, adding data
# as needed. We process them in lexicographic order!
for key in sorted(config.keys()):
config_per_source = config[key]
print "Working on source:", key
ProcessSource(db_source, db_prod, geocoder, entities_lookup,
config_per_source, test_mode)
geocoder.PrintStats()
entities_lookup.print_statistics()
# Process yaml-free sources:
process_source_rpvs(db_source, db_prod, geocoder, entities_lookup,
test_mode)
db_source.close()
# Run post processing.
# TODO: For now post processing requires access to the profil
# source schema. Remove this when fixed.
schema_profil = db_prod.get_latest_schema('source_internal_profil_')
db_prod.execute('SET search_path="' + prod_schema_name + '", "' +
schema_profil + '", public;')
post_process.do_post_processing(db_prod, test_mode)
# Create materialized view for entity search after all entities
# have been created.
db_prod.execute("""
CREATE MATERIALIZED VIEW entities_search AS
SELECT
id,
to_tsvector('simple', unaccent(name)) as search_vector
FROM entities;
CREATE INDEX ON entities_search(search_vector);
CREATE INDEX ON entities_search USING gin(search_vector);
""")
# Grant apps read-only access to the newly created schema and tables within
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'data')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'verejne')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'kataster')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'prepojenia')
db_prod.grant_usage_and_select_on_schema(prod_schema_name, 'obstaravania')
# Commit database changes and close database connections
db_address_cache.commit()
db_address_cache.close()
if test_mode:
db_prod.conn.rollback()
print('[OK] Rolled back database changes (test mode)')
else:
db_prod.commit()
db_prod.close()
__email__ = "*****@*****.**"
import os
import markupsafe
import flask
import json
import geocoder
import wikiguide
app = flask.Flask(__name__,
static_url_path='',
static_folder='static',
template_folder='templates')
_geo = geocoder.Geocoder()
_wiki_guide = wikiguide.Wikiguide()
@app.route("/", methods=['get'])
@app.route("/geofind", methods=['get'])
def find():
response = {}
items = []
pages = []
lat = flask.request.args.get("lat")
lng = flask.request.args.get("lng")
addr = flask.request.args.get("address")
if (lat and lng):
items = _wiki_guide.geosearch(lat,lng).get('query').get('geosearch')
elif (addr):
def setUpClass(cls):
super(GeocoderTest, cls).setUpClass()
db_address_cache = DatabaseConnection(
path_config='db_config_update_source.yaml',
search_path='address_cache')
cls.geocoder = geocoder_lib.Geocoder(db_address_cache, None, True)
'alternative is to use only the geocache.')
parser.add_option('-p', '--print_records', dest='print_recs',
action='store_true', default=False,
help='Set to print out records as they\'re coded.')
parser.add_option('-o', '--output_format', default='',
help='Set to either lat-lons.js, lat-lons-ny.js, records.js'
' or entries.txt to output one of these formats.')
parser.add_option('', '--lat_lon_map', default='', dest='lat_lon_map',
help='Lat/lon cluster map, built by cluster-locations.py. '
'Only used when outputting lat-lons.js')
(options, args) = parser.parse_args()
if options.geocode:
g = geocoder.Geocoder(options.use_network, 2) # 2s between geocodes
else:
g = None
geocoders = coders.registration.coderClasses()
geocoders = [coder() for coder in geocoders]
if options.ok_coders != 'all':
ok_coders = options.ok_coders.split(',')
geocoders = [c for c in geocoders if c.name() in ok_coders]
if len(geocoders) != len(ok_coders):
sys.stderr.write('Coder mismatch: %s vs %s\n' % (options.ok_coders, ','.join([c.name() for c in geocoders])))
sys.exit(1)
# TODO(danvk): does this belong here?
lat_lon_map = {}
def test_geocode():
geo = geocoder.Geocoder()
decoded = geo.get_blockgroup((47.6145, -122.3210)).item()
import os
import sqlite3
import geocoder
from datetime import datetime
gc = geocoder.Geocoder()
class Contact(object):
def __init__(self, phone):
self.number = phone
self.subscriber = False
self.addresses = []
# connect to the db
conn = sqlite3.connect('db/prod.sqlite')
c = conn.cursor()
# check if current user is actively subscribed to any addresses
n = (self.number, )
for row in c.execute(
'SELECT matched_address FROM subscribers WHERE phone=? AND active=1',
n).fetchall():
self.addresses.append(row)
conn.close()
def watch(self, address):
"""Add a new subscription or activate an existing one"""
geo = gc.geocode(address)
location = (round(geo['location']['x'],
'''
street_str = make_street_str(street)
avenue_str = make_avenue_str(avenue, street)
response = g.Locate('%s and %s, Manhattan, NY' % (street_str, avenue_str))
r = response['results'][0]
if r['types'] != ['intersection']: return None
if r.get('partial_match'): return None # may be inaccurate
loc = r['geometry']['location']
lat_lon = loc['lat'], loc['lng']
return lat_lon
if __name__ == '__main__':
g = geocoder.Geocoder(True, 1) # use network, 1s wait time.
crosses = []
# for street in range(14, 125):
# for ave in range(-3, 13):
# crosses.append((ave, street))
for street in range(1, 14):
for ave in range(-3, 7):
crosses.append((ave, street))
# random.shuffle(crosses)
for i, (ave, street) in enumerate(crosses):
lat, lon = locate(ave, street) or ('', '')
print '%d\t%d\t%s\t%s' % (street, ave, lat, lon)
# print '%d / %d --> %s / %s --> %s' % (
# 1 + i, len(crosses),
"to output one of these formats.")
parser.add_option("", "--from_cache", default="", dest="from_cache",
help="Set to a comma-separated list of coders to read " +
"them from the pickle cache instead of regenerating.")
parser.add_option("", "--write_cache", default=False, dest="write_cache",
action="store_true", help="Create pickle cache")
parser.add_option("", "--ids_filter", default="", dest="ids_filter",
help="Comma-separated list of Photo IDs to consider.")
parser.add_option("", "--lat_lon_map", default="", dest="lat_lon_map",
help="Lat/lon cluster map, built by cluster-locations.py. "
"Only used when outputting lat-lons.js")
(options, args) = parser.parse_args()
if options.geocode:
g = geocoder.Geocoder(options.maps_key, 5)
else:
g = None
geocoders = coders.registration.coderClasses()
geocoders = [coder() for coder in geocoders]
if options.ok_coders != 'all':
ok_coders = options.ok_coders.split(',')
geocoders = [c for c in geocoders if c.name() in ok_coders]
cache_coders = options.from_cache.split(',')
for idx, coder in enumerate(geocoders):
if coder.name() in cache_coders:
geocoders[idx] = coders.cached_coder.CachedCoder(coder.name())
cache = defaultdict(list)
def search_places(query='', maxprice=4, radius='2000'):
bot_response = []
query = process_input(query)
if query == ERROR_MESSAGE:
return ERROR_MESSAGE, True
api_type = closest_type(query[0])
geocode = geocoder.Geocoder(query[1])
location = geocode.get_geocode()
if location is None:
return ERROR_MESSAGE, True
formatted_location = [str(location['lat']) + ',' + str(location['lng'])]
parameters = parser.urlencode(
{
'keyword': query,
'location': formatted_location,
'region': 'gr',
'opennow': '',
'radius': radius,
'type': api_type,
'maxprice': maxprice,
'rankby': 'prominence',
'key': PLACES_API_KEY
},
doseq=True)
# performs a nearby on the specified query
req = urllib.request.Request(BASE_PLACES_URL + parameters)
response = urllib.request.urlopen(req).read()
contents = json.loads(response.decode('utf-8'))
check = checker.ApiChecker(contents, 'places')
# check status for initial request
check.check_response_status()
if not check.get_status() or check.get_zero_res_query():
return None, True
results = contents['results']
latitudes = [location['lat']]
longitudes = [location['lng']]
for result in results:
# get price level
price_level = PRICE_LEVELS[result['price_level']]
# check if there is rating
if 'rating' in result:
bot_response.append(
"Open now: {}, Place: {}, Address: {}, Price level: {} and Rating: {}"
.format(str(result['opening_hours']['open_now']),
result['name'], str(result['vicinity']), price_level,
str(result['rating'])))
else:
bot_response.append(
"Open now: {}, Place: {}, Address: {}, Price level: {}".format(
str(result['opening_hours']['open_now']), result['name'],
str(result['vicinity']), price_level))
# needed for plotter
latitudes.append(result['geometry']['location']['lat'])
longitudes.append(result['geometry']['location']['lng'])
plotter(latitudes, longitudes)
if not bot_response: # if bot_response is an empty list
return None, True
else:
return bot_response, False
from bottle import static_file,route, run, template, get , post , request from tweepy import API,Cursor,TweepError ''' Note: This code includes some parts provided by other researches ''' import geocoder geoC = geocoder.Geocoder(dataset='geonames') #Import the necessary methods from tweepy library from tweepy import OAuthHandler from tweepy import API from tweepy import TweepError #import json import langid import pickle import cPickle import numpy as np import gcd_dist import lib_grid_search import re from haversine import haversine from nltk.tokenize import TweetTokenizer tknzr = TweetTokenizer() def distance(lat1, lon1, lat2, lon2):