def get_coordinates(self): geocoder = Geocoder() uri = urlparse(self.path) query = parse_qs(uri.query) if query.get('address') is None: return None coordinates = geocoder.geocode(query['address'][0]) return coordinates
def search(self, keyword=None, city=None, state=None, coordinates=None, radius='5mi', num=20): auth = tweepy.OAuthHandler(k.consumer_key, k.consumer_secret) auth.set_access_token(k.access_token, k.access_token_secret) api = tweepy.API(auth) geo = Geocoder() print '\nGathering tweets...\n' #kcs if keyword is not None and city is not None and state is not None and coordinates is None: coordinates = geo.getCoordinates(city, state) coordinate_str = '{lat},{lng},'.format(**coordinates) + radius tweets = api.search(q=keyword, geocode=coordinate_str, count=num) #cs if keyword is None and city is not None and state is not None and coordinates is None: coordinates = geo.getCoordinates(city, state) coordinate_str = '{lat},{lng},'.format(**coordinates) + radius tweets = api.search(geocode=coordinate_str, count=num) #kl if keyword is not None and city is None and state is None and coordinates is not None: coordinate_str = '{lat},{lng},'.format(**coordinates) + radius tweets = api.search(q=keyword, geocode=coordinate_str, count=num) #l if keyword is None and city is None and state is None and coordinates is not None: coordinate_str = '{lat},{lng},'.format(**coordinates) + radius tweets = api.search(geocode=coordinate_str, count=num) #k if keyword is not None and city is None and state is None and coordinates is None: tweets = api.search(q=keyword, count=num) #st tweets.sort(reverse=True, key=lambda x: x.author.followers_count) print '\nAnalyzing data...\n' a = SentimentAnalyzer() return [{ 'author': t.author.name, "tweet": t.text, "id": t.id, "sentiment": a.analyze(t.text) } for t in tweets]
def _geocode(self, lat, lon): geo = Geocoder() df = geo.geocode_point((float(lat), float(lon))) # import pdb; pdb.set_trace() self.block_group = df[cn.BLOCK_GROUP].item() self.neighborhood_long = df[cn.NBHD_LONG].item() self.neighborhood_short = df[cn.NBHD_SHORT].item() self.council_district = df[cn.COUNCIL_DISTRICT].item() self.urban_village = df[cn.URBAN_VILLAGE].item() self.zipcode = df[cn.ZIPCODE].item() return self
def calculate_coords(self): if self.city_name is None: return geocoder = Geocoder(self.city_name) geocode = geocoder.get_geocode() if geocode is None: return self.lat = geocode["lat"] self.lon = geocode["lng"]
def main(): env = dotenv_values() db_creds = { key: env[key] for key in ["dbname", "user", "password", "host", "port"] } conn = db.connect(**db_creds) iter_conn = db.connect(**db_creds) gc = Geocoder(conn, iter_conn, debug=False) gc.geocode() conn.close() iter_conn.close()
def _handle_client(self, client, address): PACKET_SIZE = 1024 while True: print("CLIENT", client) data = client.recv(PACKET_SIZE).decode() if not data: break data_split = data.split(' ') request_method = data_split[0] if not request_method == "GET": print("Unknown HTTP request method: {method}".format( method=request_method)) else: print("Successful method used: {method}".format( method=request_method)) print("Method: {m}".format(m=request_method)) print("Request Body: {b}".format(b=data)) print("Data {data}".format(data=data_split)) http_code = 200 geocode_response = [] try: geocoder = Geocoder(query_string_data=data_split[1]) geocode_response = geocoder.request() if 'error' in geocode_response[0]: http_code = 403 except Exception as e: print("Geocoder request failed: {e}".format(e=e)) geocode_response = [{ "error": "error_initializing_geocoders" }] http_code = 500 response_header = self._generate_headers(http_code) response = response_header.encode() response += json.dumps(geocode_response) client.send(response) client.close() break
def geocode_city(self, city_state_country): client = Geocoder("http://open.mapquestapi.com/nominatim/v1/search?format=json") response = client.geocode(city_state_country) if (not response): print ("no response for city=%s" % (city_state_country)) return None #print (response) #print response[0][self.DISPLAY_NAME] #print response[0][self.BOUNDINGBOX] #print response[0][self.ADDRESS] '''return boundary of city in format [bottom, top, left, right]''' return response[0][self.BOUNDINGBOX]
def geocode_all(db, data_folder="geocoder/data", terms_folder="geocoder/terms", lines_per_insert=1000): print("Loading table...") # The query bellow seems not very efficient... # Maybe change it as the link says. # https://stackoverflow.com/questions/7389759/memory-efficient-built-in-sqlalchemy-iterator-generator non_geocoded = Execucao.query.filter(Execucao.searched == False).all() with Geocoder(data_folder, terms_folder) as geocoder: counter = ProgressCounter(len(non_geocoded), print_abs=True) to_be_inserted = 0 for row in non_geocoded: cells = get_geolocable_cells(row) geoent = geocoder.geocode_list(cells) if geoent: lat, lon, reg = geoent.best_coords() if lat: row.point = "POINT(%s %s)" % (lon, lat) row.searched = True to_be_inserted += 1 if to_be_inserted == lines_per_insert: db.session.commit() to_be_inserted = 0 counter.update() if to_be_inserted: db.session.commit() counter.end()
def geocode_all(db, data_folder="geocoder/data", terms_folder="geocoder/terms", lines_per_insert=1000): print("Loading table...") non_geocoded = get_non_geocode(lines_per_insert) if non_geocoded: while non_geocoded: with Geocoder(data_folder, terms_folder) as geocoder: counter = ProgressCounter(len(non_geocoded), print_abs=True) to_be_inserted = 0 for row in non_geocoded: cells = get_geolocable_cells(row) geoent = geocoder.geocode_list(cells) if geoent: lat, lon, reg = geoent.best_coords() if lat: row.point = "POINT(%s %s)" % (lon, lat) row.searched = True to_be_inserted += 1 if to_be_inserted == lines_per_insert: db.session.commit() to_be_inserted = 0 counter.update() if to_be_inserted: db.session.commit() counter.end() non_geocoded = get_non_geocode(lines_per_insert)
def _get_blockgroup(self, df): geo = Geocoder() temp = [] for chunk in self._chunker(df, 250): coords = chunk.loc[:, (cn.LAT, cn.LON)] blkgrps = geo.geocode_df(coords) blkgrps.columns = [ cn.LAT, cn.LON, cn.DEST_BLOCK_GROUP, cn.NBHD_LONG, cn.NBHD_SHORT, cn.COUNCIL_DISTRICT, cn.URBAN_VILLAGE, cn.ZIPCODE ] temp.append( pd.merge(chunk, blkgrps, left_on=[cn.LAT, cn.LON], right_on=[cn.LAT, cn.LON], how='left').drop_duplicates()) df = pd.concat(temp, sort=False).reset_index() return df
def __init__(self, origin_lat: float, origin_lon: float, mapquest_api_key: str) -> None: """ Creates a new monitor. Args: origin_lat (float): The origin latitude. origin_lon (float): The origin longitude. mapquest_api_key (str): The MapQuest API key for geocoding. """ self._ses = requests.Session() self._parser = RealTimeParser() self._coder = Geocoder(mapquest_api_key) self._origin = (origin_lat, origin_lon) self._incidents = {} # type: Dict[str, Incident] self._ses.headers.update({ "User-Agent": "SFD Feed Watcher " "(https://github.com/xyx0826/sea_fires_around_me)" })
def post(self): lnd = self.request.get('lnd') if lnd: geo = Geocoder(apikey=apikey) lnd_coord = geo.geocode(location=lnd) if lnd_coord: shops = Shop.all() shops = [ (shop, calc_distance(shop, lnd_coord), mapurl(shop.geo.lat, shop.geo.lon) ) for shop in shops ] shops.sort(key=lambda k: k[1]['distance']) self.render_response('mobile/search_result.html', { 'lnd': lnd, 'lnd_map': mapurl(lnd_coord['lat'], lnd_coord['lng']), 'shops': shops[0:30] }) else: self.response.out.write('not found') else: self.redirect('/m/search')
def __new_bot(self): timezone = TimezoneApi('bar') timezone.load = MagicMock(return_value=12345) darksky = WeatherSource('foo', timezone) darksky.load = MagicMock( return_value=Weather(now=WeatherAtTime(20, 'Clear', ''), day=WeatherDay(19, 21, 'Mostly Cloudy', ''))) geocoder = Geocoder('foo') geocoder.geocode = MagicMock(return_value={ 'results': [{ 'geometry': { 'location': { 'lat': 1.2, 'lng': 3.4 } } }] }) webcam_source = WebcamSource('foo') webcam_source.load = MagicMock(return_value=None) return WeatherBot(darksky, geocoder, webcam_source)
def arcgis(location, proxies='', timeout=5.0): """ Retrieves geocoding data from ArcGIS's REST geocoding API. >>> g = geocoder.arcgis('380 New York St, Redlands, California') >>> g.latlng (34.05649072776595, -117.19566584280369) >>> g.postal '92373' ... Official Docs ------------- http://resources.arcgis.com/en/help/arcgis-rest-api/ """ provider = Arcgis(location) return Geocoder(provider, proxies=proxies, timeout=timeout)
def ip(location, proxies='', timeout=5.0): """ Geocodes an IP address using MaxMind's services. >>> g = geocoder.ip('74.125.226.99') >>> g.latlng (37.4192, -122.0574) >>> g.address 'Mountain View, California United States' ... Official Docs ------------- http://www.maxmind.com/en/geolocation_landing """ provider = Ip(location) return Geocoder(provider, proxies=proxies, timeout=timeout)
def osm(location, proxies='', timeout=5.0): """ Retrieves geocoding data from OSM's data using Nominatim's geocoding API. >>> g = geocoder.osm('Tacloban City') >>> g.latlng (11.2430274, 125.0081402) >>> g.country 'Philippines' ... Official Docs ------------- http://wiki.openstreetmap.org/wiki/Nominatim """ provider = Osm(location) return Geocoder(provider, proxies=proxies, timeout=timeout)
def mapquest(location, proxies='', timeout=5.0): """ Retrieves geocoding data from MapQuest's address geocoding API. >>> g = geocoder.mapquest('1555 Blake street, Denver') >>> g.latlng (39.740009, -104.992264) >>> g.quality 'CITY' ... Official Docs ------------- http://www.mapquestapi.com/geocoding/ """ provider = Mapquest(location) return Geocoder(provider, proxies=proxies, timeout=timeout)
def bing(location, key='', proxies='', timeout=5.0): """ Retrieves geocoding data from Bing's REST location API. >>> key = 'XXXXX' >>> g = geocoder.bing('Medina, Washington', key=key) >>> g.latlng (47.615821838378906, -122.23892211914062) >>> g.country 'United States' ... Official Docs ------------- http://msdn.microsoft.com/en-us/library/ff701714.aspx """ provider = Bing(location, key=key) return Geocoder(provider, proxies=proxies, timeout=timeout)
def tomtom(location, key='', proxies='', timeout=5.0): """ Retrieves geocoding data from TomTom's geocoding API. >>> key = 'XXXXX' >>> g = geocoder.tomtom('Amsterdam, Netherlands', key=key) >>> g.latlng (52.373166, 4.89066) >>> g.quality 'city' ... Official Docs ------------- http://developer.tomtom.com/products/geocoding_api """ provider = Tomtom(location, key=key) return Geocoder(provider, proxies=proxies, timeout=timeout)
def reverse(latlng, proxies='', timeout=5.0): """ Reverse geocodes a location based on Lat & Lng inputs using Google's reverse geocoding API V3. >>> latlng = (37.4192, -122.0574) >>> g = geocoder.reverse(latlng) >>> g.address 'Sevryns Road, Mountain View, CA 94043, USA' >>> g.postal '94043' ... Official Docs ------------- https://developers.google.com/maps/documentation/geocoding/ """ provider = Reverse(latlng) return Geocoder(provider, proxies=proxies, timeout=timeout)
def nokia(location, app_id='', app_code='', proxies='', timeout=5.0): """ Retrieves geocoding data from Nokia's HERE geocoder API. >>> app_id = 'XXXXX' >>> app_code = 'XXXXX' >>> g = geocoder.nokia('Keilaniemi, Espoo') >>> g.latlng (60.1759338, 24.8327808) >>> g.country 'FIN' ... Official Docs ------------- https://developer.here.com/rest-apis/documentation/geocoder """ provider = Nokia(location, app_id=app_id, app_code=app_code) return Geocoder(provider, proxies=proxies, timeout=timeout)
def geonames(location, username='', proxies='', timeout=5.0): """ Retrieves geocoding data from Geonames's Web Service API. >>> username = '******' >>> g = geocoder.geonames('Springfield, Virginia', username=username) >>> g.latlng (38.78928, -77.1872) >>> g.country 'United States' >>> g.population 30484 ... Official Docs ------------- http://www.geonames.org/export/web-services.html """ provider = Geonames(location, username=username) return Geocoder(provider, proxies=proxies, timeout=timeout)
def check_input(self, location): lat, lng = 0.0, 0.0 # Checking for a String if isinstance(location, str): lat, lng = Geocoder(Google(location)).latlng # Checking for List of Tuple if isinstance(location, (list, tuple)): lat, lng = self.check_for_list(location) # Checking for Dictionary elif isinstance(location, dict): lat, lng = self.check_for_dict(location) # Checking for a Geocoder Class elif hasattr(location, 'latlng'): lat, lng = location.latlng # Return Results return lat, lng
def google(location, client='', secret='', proxies='', api_key='', timeout=5.0): """ Retrieves geocoding data from Google's geocoding API V3 >>> g = geocoder.google('1600 Amphitheatre Pkwy, Mountain View, CA') >>> g.latlng (37.784173, -122.401557) >>> g.country 'United States' ... Official Docs ------------- https://developers.google.com/maps/documentation/geocoding/ """ provider = Google(location, client=client, secret=secret, api_key=api_key) return Geocoder(provider, proxies=proxies, timeout=timeout)
class PostalCodeParser: def columnNames(self): return [ COUNTRY_CODE, NAME, # really zipcode PLACE_NAME, ADMIN1_NAME, ADMIN1_CODE, ADMIN2_NAME, ADMIN2_CODE, ADMIN3_NAME, ADMIN3_CODE, LATITUDE, LONGITUDE, ACCURACY ] def __init__(self, filename): self.csvReader = unicode_csv_reader(open(filename), delimiter='\t', quotechar = '\x07') self.lastRow = self.csvReader.next() self.count = 0 self.geocoder = Geocoder() self.done = None def makeFeature(self, row): if len(row) != len(self.columnNames()): print row print("col length mismatch: %d vs %d" % (len(row), len(self.columnNames()))) values = {} for (key, val) in zip(self.columnNames(), row): values[key] = val return Feature(values) # make this a real iterator? def getNextFeatureSet(self): first_feature = self.makeFeature(self.lastRow) self.lastRow = None features = [first_feature] for row in self.csvReader: self.count += 1 if self.count % 1000 == 0: print "imported %d zips so far" % self.count self.lastRow = row feature = self.makeFeature(row) if feature.all_names() == first_feature.all_names(): features.append(feature) else: break if self.lastRow is None: self.done = True return features # try to find each of the admin codes # if we can't find it, create and return them def find_and_create_parents(self, feature): def tooFar(candidate): km = distance_km(feature.latitude(), feature.longitude(), candidate[0]['lat'], candidate[0]['lng']) return km < 20000 # try to find CC, then admin1, then admin2, then admin3, then place_name # if we find it and the name disagrees, add that # if we can't find it, try geocoding for it instead cc = feature.country_code() parents = [feature.cc_id()] def tryGeocoder(query, record): parents = [] (geocodes, meta) = self.geocoder.geocode(query) geocodes = filter(tooFar, geocodes) if len(meta['query']) == 0 and len(geocodes) > 0: for g in geocodes: parents.append(g[0]['featureid']) if 'parents' in g[0]: parents += g[0]['parents'] else: record['parents'] = list(set(filter(None, record['parents']))) geonames.insert(record) parents.append(record['featureid']) return parents fcodes = [ADM1, ADM2, ADM3] codes = [ feature.admin1_id(), feature.admin2_id(), feature.admin3_id() ] names = [ feature.values[ADMIN1_NAME], feature.values[ADMIN2_NAME], feature.values[ADMIN3_NAME] ] for i in xrange(0, len(codes)): code = codes[i] name = names[i] if not code: continue geocodes = [g for g in geonames.find({'featureid': code, 'cc': cc, 'parents': { '$all': filter(None,codes[0:i]) + [feature.cc_id()] } })] if len(geocodes): parents.append(code) for g in geocodes: if normalize(name) not in g['names']: geonames.update({'_id': g['_id']}, {'$addToSet': { 'names': normalize(name) }}, safe=True) else: query = ' '.join(names[0:i+1]) + ' ' + cc parents += tryGeocoder(query, { 'cc': feature.country_code(), 'feature_code': 'A', 'feature_class': fcodes[i], 'featureid': code, 'names': [normalize(name)], 'lat': feature.latitude(), 'lng': feature.longitude(), 'parents': filter(None, parents), }) # looking for place name query = feature.values[PLACE_NAME] + ' ' + ' '.join(names) + ' ' + cc parents += tryGeocoder(query, { 'cc': feature.country_code(), 'feature_code': 'P', 'feature_class': 'PPL', 'featureid': normalize(feature.values[PLACE_NAME]), 'names': [normalize(feature.values[PLACE_NAME])], 'lat': feature.latitude(), 'lng': feature.longitude(), 'parents': filter(None, parents), 'displayNames': [{ 'l': 'en', 's': feature.name(), 'p': True }] }) return list(set(parents)) def parse(self): while not self.done: featureSet = self.getNextFeatureSet() parents = [] last_feature = None for feature in featureSet: if last_feature and last_feature.values[PLACE_NAME] == feature.values[PLACE_NAME]: continue if feature.country_code() == 'PT': next parents = feature.parents() all_names = feature.all_names() # with better run-time ranking, we could do this instead of synthesizing parents # all_names.append(last_feature.values[PLACE_NAME]) if options.create_missing_parents: parents += self.find_and_create_parents(feature) last_feature = feature # save the first feature feature = featureSet[0] try: record = { 'featureid': '%s-%s' % (feature.country_code(), all_names[0]), 'cc': feature.country_code(), 'feature_code': 'Z', 'feature_class': 'ZIP', 'names': [normalize(n) for n in all_names], 'lat': feature.latitude(), 'lng': feature.longitude(), 'parents': list(set(parents)), 'accuracy': feature.accuracy(), 'displayNames': [{ 'l': 'en', 's': feature.name(), 'p': True }] } except: error(values, row) geonames.insert(record)
def get_grid_with_address(self, address, width_m, height_m, grid_spacing_m): """Return a grid of points centered on <address>.""" geocoder = Geocoder() lat_lon, details = geocoder.get_location_by_address(address) lat = lat_lon[0]; lon = lat_lon[1]; return self.get_grid_with_center(lat, lon, width_m, height_m, grid_spacing_m)
import sys from geocoder import Geocoder if len(sys.argv) < 4: print "[USAGE]: python %s [state abbr filepath] [city filepath] [location text filepath]" % sys.argv[0] exit() state_abbr_filepath = sys.argv[1] city_filepath = sys.argv[2] location_text_filepath = sys.argv[3] gc = Geocoder(state_abbr_filepath, city_filepath) for line in open(location_text_filepath, 'r'): location_text = line.rstrip() point = gc.geocode(location_text) if point == None: print None else: print point[0], point[1]
return d source = fiona.open(sys.argv[1]) output_schema = source.schema.copy() output_schema['properties']['db_geonameid'] = 'str' output_schema['properties']['db_lat'] = 'float' output_schema['properties']['db_lng'] = 'float' output_schema['properties']['db_distance'] = 'float' output = fiona.open('fixed-' + sys.argv[1], 'w', crs=source.crs, driver=source.driver, schema=output_schema) geocoder = Geocoder('demo.twofishes.net') for f in source: query = u"%s %s %s" % ( f['properties']['NAME'], f['properties']['ADM1NAME'], f['properties']['ADM0NAME'] ) print query ll = (f['properties']['LATITUDE'], f['properties']['LONGITUDE']) llStr = '%s,%s' % ll hint = { 'woeHint': 'TOWN', 'll': llStr }
class RealTime: """ A real-time 911 dispatch monitor. """ def __init__(self, origin_lat: float, origin_lon: float, mapquest_api_key: str) -> None: """ Creates a new monitor. Args: origin_lat (float): The origin latitude. origin_lon (float): The origin longitude. mapquest_api_key (str): The MapQuest API key for geocoding. """ self._ses = requests.Session() self._parser = RealTimeParser() self._coder = Geocoder(mapquest_api_key) self._origin = (origin_lat, origin_lon) self._incidents = {} # type: Dict[str, Incident] self._ses.headers.update({ "User-Agent": "SFD Feed Watcher " "(https://github.com/xyx0826/sea_fires_around_me)" }) def _get_two_day_rows(self) -> "List[IncidentRow]": """ Gets all incident rows from today and yesterday to deal with midnight jumps. Returns: List[IncidentRow]: A list of all active incident rows. """ r = self._ses.get(REALTIME_ENDPOINT, params={"action": "Today"}) self._parser.feed(r.text) rows = self._parser.get_rows() yesterday = datetime.today() - timedelta(days=1) r = self._ses.get(REALTIME_ENDPOINT, params={"incDate": yesterday.strftime("%m/%d/%Y")}) self._parser.feed(r.text) yesterday_rows = self._parser.get_rows() if len(yesterday_rows) > 0: rows += yesterday_rows return rows def _add_incident(self, row: IncidentRow) -> None: """ Adds a new incident and prints a message. Args: row (IncidentRow): The new incident. """ latlon = self._coder.geocode(row.loc) inc = Incident(row.id, row.datetime, row.loc, latlon, row.typ) dist = inc.get_dist_to(self._origin) dire = inc.get_direction_to(self._origin) print(f"Incident of type {inc.get_type()} is opened " f"at {inc.get_addr()}, {dist:.2f} km {dire}.") self._incidents[row.id] = inc def _update_incident(self, row: IncidentRow) -> None: """ Updates an open incident. Args: row (IncidentRow): The incident to update. """ inc = self._incidents[row.id] for unit in row.units.split(" "): inc.add_unit(unit) inc.update_type(row.typ) def _remove_incident(self, inc_id: str) -> None: """ Removes a resolved incident and prints a message. Args: inc_id (str): The ID of the resolved incident. """ inc = self._incidents[inc_id] addr = inc.get_addr() mins = inc.get_time_since().total_seconds() / 60 print(f"Incident at {addr} is now resolved after {int(mins)} minutes.") del self._incidents[inc_id] def update(self) -> None: """ Checks if there are incidents to update. """ rows = self._get_two_day_rows() # Add new rows for row in rows: if row.id not in self._incidents.keys(): self._add_incident(row) # Update existing rows updated_rows = [r for r in rows if r.id in self._incidents.keys()] for updated_row in updated_rows: self._update_incident(updated_row) # Remove deleted rows active_ids = [r.id for r in rows] for inc_id in list(self._incidents): if inc_id not in active_ids: self._remove_incident(inc_id)
def text(): """Respond to incoming texts with a message.""" resp = MessagingResponse() # get sender phone number, check if they are a current subscriber incoming_number = request.values.get('From')[2:] if incoming_number in users.keys(): caller = users[incoming_number] else: caller = contact.Contact(incoming_number) # get body of incoming SMS and format it body = request.values.get('Body') b = body.upper().strip() # check if the body is 'HEALTH', 'EDU', 'ADD', 'REMOVE' or anything else if b in keywords['Health']: health_msg = message.HealthMsg().make_msg() resp.message(health_msg) print("{} texted HEALTH".format(incoming_number)) myLogger.emit(logging.INFO, "HEALTH texted", request.values) elif b in keywords['Edu']: call_msg = message.CallMsg().make_msg() resp.message(call_msg) print("{} requested a call from a Health Educator".format( incoming_number)) myLogger.emit(logging.INFO, "Health Educator call requested", request.values) # get users last requested address or the address they're subscribed to if incoming_number in users.keys(): edu_addr = caller.last_requested_address else: edu_addr = [a[0] for a in caller.addresses][0] # calculate demos nearby to generate short report for Health Dept located = Geocoder().geocode(edu_addr) # query Socrata datasets scheduled_demos = soda_client.get( "tsqq-qtet", where="within_circle(location, {}, {}, 155)".format( located['location']['y'], located['location']['x'])) pipeline_demos = soda_client.get( "dyp9-69zf", where="within_circle(location, {}, {}, 155)".format( located['location']['y'], located['location']['x'])) past_demos = soda_client.get( "rv44-e9di", where="within_circle(location, {}, {}, 155)".format( located['location']['y'], located['location']['x'])) upcoming_demos = scheduled_demos + pipeline_demos # format 'em demo_dates = [] if len(scheduled_demos) > 0: for d in scheduled_demos: demo_date = "{}".format( datetime.strptime( (d['demolish_by_date']), '%Y-%m-%dT%H:%M:%S').strftime('%m-%d-%Y')) demo_dates.append(demo_date) else: demo_date = None demo_dates.append(demo_date) # send log to Loggly log_extras = { "last_address_texted": edu_addr, "upcoming_demos_count": len(upcoming_demos), "past_demos_count": len(past_demos), "next_knockdown_date": demo_dates[0] } myLogger.emit(logging.INFO, "Health Educator call requested.", request.values, last_address_texted=edu_addr, upcoming_demo_count=len(upcoming_demos), next_knockdown_date=demo_dates[0]) # send request to Slack webhook_url = os.environ['SLACK_WEBHOOK_URL'] caller_msg = ":phone: `{}` requested a call from a Health Educator \nLast address texted: *{}* \nNumber of upcoming demos: *{}* \nNumber of past demos: *{}* \nNext knock-down date: *{}*".format( incoming_number, edu_addr, len(upcoming_demos), len(past_demos), demo_dates[0]) slack_data = {'text': caller_msg} response = requests.post(webhook_url, data=json.dumps(slack_data), headers={'Content-Type': 'application/json'}) if response.status_code != 200: myLogger.emit(logging.ERROR, "Slack request failed", [], status_code=response.status_code, response=response.text) raise ValueError( 'Request to slack returned an error %s, the response is:\n%s' % (response.status_code, response.text)) elif b in keywords['Add'] and caller.last_requested_address: caller.watch(caller.last_requested_address) msg = message.SubscribeMsg(caller.last_requested_address) success_msg = msg.make_msg() resp.message(success_msg) # remove from users so we grab a 'fresh' copy of the user with sheet rows del users[incoming_number] print("{} subscribed to {}".format(incoming_number, caller.last_requested_address)) myLogger.emit(logging.INFO, "Caller subscribed to an address", request.values, address=caller.last_requested_address) elif b in keywords['Remove']: for address in caller.addresses: caller.unwatch(address) msg = message.UnsubscribeMsg([a[0] for a in caller.addresses]) remove_msg = msg.make_msg() resp.message(remove_msg) print("{} unsubscribed from {} addresses".format( incoming_number, len(caller.addresses))) myLogger.emit(logging.INFO, "Caller unsubscribed to a number of addresses", request.values, address_count=len(caller.addresses)) else: pattern = re.compile('(\d{2,5})\s?(\w{2,})') result = pattern.search(body) if result: send = result.group(1) + ' ' + result.group(2) else: send = body # send it to the geocoder located = Geocoder().geocode(send) # if it's a valid address, build up a text message with demos nearby if located: print("Geocoded {} from {}".format(located['address'], incoming_number)) myLogger.emit(logging.INFO, "Geocoded an address", request.values, geocode=located["address"]) msg = message.DemoMsg(located) demo_msg = msg.make_msg() resp.message(demo_msg) # store matched address caller.last_requested_address = located['address'][:-7] users[incoming_number] = caller # default message for a bad address else: default_msg = message.DefaultMsg().make_msg() resp.message(default_msg) print("Couldn't geocode '{}' from {}; Sent it to Slack".format( body, incoming_number)) myLogger.emit(logging.ERROR, "Geocoding failed", request.values, from_address=body) # send it to Slack webhook_url = os.environ['SLACK_WEBHOOK_URL'] err_msg = ":exclamation: demo-alerts can't geocode `{}` from `{}`".format( send, incoming_number) slack_data = {'text': err_msg} response = requests.post( webhook_url, data=json.dumps(slack_data), headers={'Content-Type': 'application/json'}) if response.status_code != 200: myLogger.emit(logging.ERROR, "Slack request failed", [], status_code=response.status_code, response=response.text) raise ValueError( 'Request to slack returned an error %s, the response is:\n%s' % (response.status_code, response.text)) # send the text return str(resp)
def main(): #TODO: make this a callback function for verification if options.geocode and not options.geonames_username: logger.error("Please enter a Geonames username to use geocoding") sys.exit(0) directory = options.directory + os.sep output_path = directory + os.sep + "rf2zoo" + os.sep output_path = os.path.normpath(output_path) if not os.path.isdir(output_path): os.mkdir(output_path) logger.info("Writing to %s" % output_path) all_folders = glob.glob(directory + "*") if options.geocode: from geocoder import Geocoder geocoder = Geocoder(options.geonames_username, options.skip_cache) for folder in all_folders: if not os.path.isdir(folder): continue #tODO: allow support for specifying the edge file directly # Check valid format of x:y where x, y are valid ASNs (_, data_folder) = os.path.split(folder) if not re.match("\d+:\d+", data_folder): continue # Only do internal topologies if options.intra_as: (asn_a, asn_b) = data_folder.split(":") if asn_a != asn_b: continue logger.info("Processing %s" % data_folder) G = nx.Graph() #TODO: check on bidirectionality #TODO: download file if not present #TODO: add pyparsing to topzootools dependencies for Pypi # Definitions for parsing colon = Literal(":").suppress() #TODO: see if comma built in comma = Literal(",").suppress() arrow = Literal("->").suppress() ASN = Word(nums) ObsCount = Word(nums) # Can have . eg St. Louis place_name = Group( OneOrMore(Word(alphas+".")) + Optional(comma) + Optional(Word(alphas))) node = Group(ASN + colon + place_name) entry = (node + arrow + node + ObsCount) filename = folder + os.sep + "edges" f = open(filename) #ToDO; print unparsed lines #ToDo: push all into dict and use add_nodes_from and add_edges_from for line in f: #print line processed = entry.parseString(line) #print processed (src, dst, obs_count) = processed src_asn, src_place = src dst_asn, dst_place = dst src_place = " ".join(src_place) dst_place = " ".join(dst_place) # Use simple string for ID (not list returned from Pyparsing which # don't always hash the same src_id = "%s %s" % (src_place, src_asn) dst_id = "%s %s" % (dst_place, dst_asn) # use full hash for node id G.add_node(src_id, ASN=int(src_asn), label =src_place ) G.add_node(dst_id, ASN=int(dst_asn), label=dst_place ) G.add_edge(src_id, dst_id, obs_count = int(obs_count)) f.close() # Relabel nodes to have id based on index if options.geocode: G = geocoder.geocode(G) geocoder.save_cache() G.name = data_folder G.graph['Creator'] = 'rocketfuel2zoo' out_file = output_path + os.sep + data_folder.replace(":", "_") + ".gml" #import pprint #pprint.pprint( sorted(G.nodes())) #pprint.pprint( G.nodes(data=True)) nx.write_gml(G, out_file)
def __init__(self, filename): self.csvReader = unicode_csv_reader(open(filename), delimiter='\t', quotechar = '\x07') self.lastRow = self.csvReader.next() self.count = 0 self.geocoder = Geocoder() self.done = None
def set_providers_to_geocoder(): providers = Provider.objects.order_by('order') global geocoder geocoder = Geocoder() for provider in providers: geocoder.add_provider(provider)
# reflect an existing database into a new model Base = automap_base() # reflect the tables Base.prepare(engine, reflect=True) # Save references to each table address_data = Base.classes.Address_Data census_data = Base.classes.Census_Data # Create the locations dataframe with all data, including both statuses loc_df = pd.read_sql_table("Address_Data", con=engine) # Create a locations dataframe with reference/seeded/locked locations ref_loc_df = loc_df[loc_df.Status == "S"] ref_loc_df = ref_loc_df.loc[:, ["Latitude", "Longitude", "StreetAddress"]] # Call the geocoder class geo = Geocoder(ref_loc_df) ################################################# # Flask Routes ################################################# # Route to render index.html template using data from db @app.route("/") def index(): """Returns all street names to populate street name dropdown in HTML for selection""" # select only StreetAddress and get rid of the number and return only unique values select_st = 'select distinct right("StreetAddress", (length("StreetAddress") - position(\' \' in "StreetAddress"))) "StreetAddress"\ from "Address_Data" a'
import config import prompts import requests from db_handler import DB from geocoder import Geocoder from datetime import timedelta from flask import Flask, request, session, render_template, url_for from twilio.twiml.messaging_response import Body, Media, Message, MessagingResponse db = DB() app = Flask(__name__) app.secret_key = config.secret_key app.config.from_object(__name__) app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(minutes=5) geocoder = Geocoder(config.api_key) @app.route('/', methods=['GET']) def root(): db.execute(config.db_credentials, config.select_all_records) data = db.get_all_records() return render_template('index.html', data=data) @app.route('/map', methods=['GET']) def map(): db.execute(config.db_credentials, config.select_all_records) data = db.get_all_records() return render_template('map.html', data=data)
import timeit from geocoder import Geocoder g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin') print g.address_to_latlng('10 WEST 49 STREET') print g.address_to_latlng('10 WEST 49 ST') print g.address_to_latlng('10 W 49 ST') print timeit.timeit("""g.address_to_latlng('99 BROADWAY')""", setup="""from geocoder import Geocoder g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')""") ''' print timeit.timeit("""g.latlng_to_address((40.769083827890107,-73.952477804528087))""",setup="""from geocoder import Geocoder g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')""") '''
import timeit from geocoder import Geocoder g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin') print g.address_to_latlng('10 WEST 49 STREET') print g.address_to_latlng('10 WEST 49 ST') print g.address_to_latlng('10 W 49 ST') print timeit.timeit("""g.address_to_latlng('99 BROADWAY')""",setup="""from geocoder import Geocoder g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')""") ''' print timeit.timeit("""g.latlng_to_address((40.769083827890107,-73.952477804528087))""",setup="""from geocoder import Geocoder g = Geocoder('manhattan-addresses.bin', 'manhattan-kdtree.bin')""") '''
from geocoder import Geocoder from query_parser import QueryParser PORT = 9000 RESPONSES = [ "Hrm... How about asking me about the weather?", "I don't know about that... Want to know about the weather?", "I'm not really into that type of thing. Let's talk about the weather.", "You must be off your rocker at the moment. Come back when you want to talk weather." ] app = Flask(__name__) CORS(app) store = {} geocoder = Geocoder() @app.route('/chat/messages', methods=['GET', 'POST']) def chat_messages(): if request.form['action'] == 'join': store[request.form['user_id']] = { "messages": ["Hello %s!" % (request.form['name'])] } reply = store[request.form['user_id']]['messages'][-1] elif request.form['action'] == 'message': location = QueryParser(request.form['text']).location() if location: lat, lng = geocoder.encode(location) reply = Forecaster(lat, lng).weather_summary()
def main(): network_files = [] if options.file: network_files.append(options.file) if options.directory: #TODO: make this os.path.join across all zootools to remove need for trailing / network_files = glob.glob(options.directory + "*.graphml") if len(network_files) == 0: logger.warn("No files found. Specify -f file or -d directory") sys.exit(0) if options.directory: path = options.directory elif options.file: path = os.path.split(options.file)[0] # Get full path - don't want to create in root dir path = os.path.abspath(path) git_version = get_git_version(path) topzootools_version = pkg_resources.get_distribution("TopZooTools").version if options.output_dir: output_path = options.output_dir else: output_path = path + os.sep + "zoogml" if options.geocode: output_path += "_geocoded" if not os.path.isdir(output_path): os.mkdir(output_path) # clean up path output_path = os.path.normpath(output_path) #TODO: check nx write pickle uses cPickle #TODO: make this optional output_pickle_path = output_path + os.sep + "cache" if not os.path.isdir(output_pickle_path): os.mkdir(output_pickle_path) geocoder = Geocoder(options.geocode, options.skip_cache) #output_path += strftime("%Y%m%d_%H%M%S") logger.info("Saving to folder: %s" % output_path) # and create cache directory for pickle files pickle_dir = path + os.sep + "cache" if not os.path.isdir(pickle_dir): os.mkdir(pickle_dir) #ToDO: check why sorting for source_file in sorted(network_files): # Extract name of network from file path filename = os.path.split(source_file)[1] net_name = os.path.splitext(filename)[0] logger.info( "Converting {0}".format(net_name)) pickle_file = "{0}/{1}.pickle".format(pickle_dir, net_name) if (os.path.isfile(pickle_file) and os.stat(source_file).st_mtime < os.stat(pickle_file).st_mtime): # Pickle file exists, and source_file is older graph = nx.read_gpickle(pickle_file) else: # No pickle file, or is outdated graph = nx.read_graphml(source_file) nx.write_gpickle(graph, pickle_file) graph = convert_to_undirected(graph) # Check for self loops for n, data in graph.nodes(data=True): if n in graph.neighbors(n): logger.warn( "Self loop {0} {1}".format(data['label'], n)) # if all nodes in network are internal, remove redundant internal tag # this makes checking if internal too complex, keep tag for all nodes """ if all(data['Internal'] == 1 for n,data in graph.nodes(data=True)): # all internal, remove tag for n in graph.nodes(): del graph.node[n]['Internal'] """ #********************************* # Geocoding (optional) #********************************* if options.geocode: graph = geocoder.geocode(graph) geocoder.save_cache() #********************************* # Remove graphics data #********************************* # Will be overwritten with name from metadata if present network_label = net_name # extract edge data for src, dst, key, data in graph.edges(data=True, keys=True): if data.get("label"): label = data['label'] extracted_data = extract_edge_data(label) # Replace the edge data with extracted graph.edge[src][dst][key] = extracted_data # remove empty note if 'Note' in graph.graph and graph.graph['Note'] == '': del graph.graph['Note'] # Strip & as yEd fails on it #TODO: use html entitites fn for this network_label_clean = network_label.replace("&", "and") # Set other graph attributes graph.graph['SourceGitVersion'] = git_version graph.graph['Creator'] = "Topology Zoo Toolset" graph.graph['ToolsetVersion'] = topzootools_version graph.graph['label'] = network_label_clean #********************************* #OUTPUT - Write the graphs to files #********************************* filename = network_label.strip() #filename = filename.title() filename = filename.replace(" ", "_") pattern = re.compile('[\W_]+') filename = pattern.sub('', filename) # And also the pickle cache - write first so older timestamp pickle_out_file = "{0}/{1}.pickle".format(output_pickle_path, filename) nx.write_gpickle(graph, pickle_out_file) gml_file = "{0}/{1}.gml".format(output_path, filename) nx.write_gml(graph, gml_file) logger.info("Wrote to %s" % gml_file)