def main(): env = dotenv_values() db_creds = { key: env[key] for key in ["dbname", "user", "password", "host", "port"] } conn = db.connect(**db_creds) iter_conn = db.connect(**db_creds) gc = Geocoder(conn, iter_conn, debug=False) gc.geocode() conn.close() iter_conn.close()
def get_coordinates(self): geocoder = Geocoder() uri = urlparse(self.path) query = parse_qs(uri.query) if query.get('address') is None: return None coordinates = geocoder.geocode(query['address'][0]) return coordinates
def geocode_city(self, city_state_country): client = Geocoder("http://open.mapquestapi.com/nominatim/v1/search?format=json") response = client.geocode(city_state_country) if (not response): print ("no response for city=%s" % (city_state_country)) return None #print (response) #print response[0][self.DISPLAY_NAME] #print response[0][self.BOUNDINGBOX] #print response[0][self.ADDRESS] '''return boundary of city in format [bottom, top, left, right]''' return response[0][self.BOUNDINGBOX]
def post(self): lnd = self.request.get('lnd') if lnd: geo = Geocoder(apikey=apikey) lnd_coord = geo.geocode(location=lnd) if lnd_coord: shops = Shop.all() shops = [ (shop, calc_distance(shop, lnd_coord), mapurl(shop.geo.lat, shop.geo.lon) ) for shop in shops ] shops.sort(key=lambda k: k[1]['distance']) self.render_response('mobile/search_result.html', { 'lnd': lnd, 'lnd_map': mapurl(lnd_coord['lat'], lnd_coord['lng']), 'shops': shops[0:30] }) else: self.response.out.write('not found') else: self.redirect('/m/search')
def __new_bot(self): timezone = TimezoneApi('bar') timezone.load = MagicMock(return_value=12345) darksky = WeatherSource('foo', timezone) darksky.load = MagicMock( return_value=Weather(now=WeatherAtTime(20, 'Clear', ''), day=WeatherDay(19, 21, 'Mostly Cloudy', ''))) geocoder = Geocoder('foo') geocoder.geocode = MagicMock(return_value={ 'results': [{ 'geometry': { 'location': { 'lat': 1.2, 'lng': 3.4 } } }] }) webcam_source = WebcamSource('foo') webcam_source.load = MagicMock(return_value=None) return WeatherBot(darksky, geocoder, webcam_source)
class PostalCodeParser: def columnNames(self): return [ COUNTRY_CODE, NAME, # really zipcode PLACE_NAME, ADMIN1_NAME, ADMIN1_CODE, ADMIN2_NAME, ADMIN2_CODE, ADMIN3_NAME, ADMIN3_CODE, LATITUDE, LONGITUDE, ACCURACY ] def __init__(self, filename): self.csvReader = unicode_csv_reader(open(filename), delimiter='\t', quotechar = '\x07') self.lastRow = self.csvReader.next() self.count = 0 self.geocoder = Geocoder() self.done = None def makeFeature(self, row): if len(row) != len(self.columnNames()): print row print("col length mismatch: %d vs %d" % (len(row), len(self.columnNames()))) values = {} for (key, val) in zip(self.columnNames(), row): values[key] = val return Feature(values) # make this a real iterator? def getNextFeatureSet(self): first_feature = self.makeFeature(self.lastRow) self.lastRow = None features = [first_feature] for row in self.csvReader: self.count += 1 if self.count % 1000 == 0: print "imported %d zips so far" % self.count self.lastRow = row feature = self.makeFeature(row) if feature.all_names() == first_feature.all_names(): features.append(feature) else: break if self.lastRow is None: self.done = True return features # try to find each of the admin codes # if we can't find it, create and return them def find_and_create_parents(self, feature): def tooFar(candidate): km = distance_km(feature.latitude(), feature.longitude(), candidate[0]['lat'], candidate[0]['lng']) return km < 20000 # try to find CC, then admin1, then admin2, then admin3, then place_name # if we find it and the name disagrees, add that # if we can't find it, try geocoding for it instead cc = feature.country_code() parents = [feature.cc_id()] def tryGeocoder(query, record): parents = [] (geocodes, meta) = self.geocoder.geocode(query) geocodes = filter(tooFar, geocodes) if len(meta['query']) == 0 and len(geocodes) > 0: for g in geocodes: parents.append(g[0]['featureid']) if 'parents' in g[0]: parents += g[0]['parents'] else: record['parents'] = list(set(filter(None, record['parents']))) geonames.insert(record) parents.append(record['featureid']) return parents fcodes = [ADM1, ADM2, ADM3] codes = [ feature.admin1_id(), feature.admin2_id(), feature.admin3_id() ] names = [ feature.values[ADMIN1_NAME], feature.values[ADMIN2_NAME], feature.values[ADMIN3_NAME] ] for i in xrange(0, len(codes)): code = codes[i] name = names[i] if not code: continue geocodes = [g for g in geonames.find({'featureid': code, 'cc': cc, 'parents': { '$all': filter(None,codes[0:i]) + [feature.cc_id()] } })] if len(geocodes): parents.append(code) for g in geocodes: if normalize(name) not in g['names']: geonames.update({'_id': g['_id']}, {'$addToSet': { 'names': normalize(name) }}, safe=True) else: query = ' '.join(names[0:i+1]) + ' ' + cc parents += tryGeocoder(query, { 'cc': feature.country_code(), 'feature_code': 'A', 'feature_class': fcodes[i], 'featureid': code, 'names': [normalize(name)], 'lat': feature.latitude(), 'lng': feature.longitude(), 'parents': filter(None, parents), }) # looking for place name query = feature.values[PLACE_NAME] + ' ' + ' '.join(names) + ' ' + cc parents += tryGeocoder(query, { 'cc': feature.country_code(), 'feature_code': 'P', 'feature_class': 'PPL', 'featureid': normalize(feature.values[PLACE_NAME]), 'names': [normalize(feature.values[PLACE_NAME])], 'lat': feature.latitude(), 'lng': feature.longitude(), 'parents': filter(None, parents), 'displayNames': [{ 'l': 'en', 's': feature.name(), 'p': True }] }) return list(set(parents)) def parse(self): while not self.done: featureSet = self.getNextFeatureSet() parents = [] last_feature = None for feature in featureSet: if last_feature and last_feature.values[PLACE_NAME] == feature.values[PLACE_NAME]: continue if feature.country_code() == 'PT': next parents = feature.parents() all_names = feature.all_names() # with better run-time ranking, we could do this instead of synthesizing parents # all_names.append(last_feature.values[PLACE_NAME]) if options.create_missing_parents: parents += self.find_and_create_parents(feature) last_feature = feature # save the first feature feature = featureSet[0] try: record = { 'featureid': '%s-%s' % (feature.country_code(), all_names[0]), 'cc': feature.country_code(), 'feature_code': 'Z', 'feature_class': 'ZIP', 'names': [normalize(n) for n in all_names], 'lat': feature.latitude(), 'lng': feature.longitude(), 'parents': list(set(parents)), 'accuracy': feature.accuracy(), 'displayNames': [{ 'l': 'en', 's': feature.name(), 'p': True }] } except: error(values, row) geonames.insert(record)
import sys from geocoder import Geocoder if len(sys.argv) < 4: print "[USAGE]: python %s [state abbr filepath] [city filepath] [location text filepath]" % sys.argv[0] exit() state_abbr_filepath = sys.argv[1] city_filepath = sys.argv[2] location_text_filepath = sys.argv[3] gc = Geocoder(state_abbr_filepath, city_filepath) for line in open(location_text_filepath, 'r'): location_text = line.rstrip() point = gc.geocode(location_text) if point == None: print None else: print point[0], point[1]
query = u"%s %s %s" % ( f['properties']['NAME'], f['properties']['ADM1NAME'], f['properties']['ADM0NAME'] ) print query ll = (f['properties']['LATITUDE'], f['properties']['LONGITUDE']) llStr = '%s,%s' % ll hint = { 'woeHint': 'TOWN', 'll': llStr } try: geocode = geocoder.geocode(query, hint) if not geocode: query = u"%s %s" % ( f['properties']['ADM1NAME'], f['properties']['ADM0NAME'] ) geocode = geocoder.geocode(query, hint) except: next f['properties']['db_geonameid'] = '' f['properties']['db_lat'] = '' f['properties']['db_lng'] = '' f['properties']['db_distance'] = 0.0
class RealTime: """ A real-time 911 dispatch monitor. """ def __init__(self, origin_lat: float, origin_lon: float, mapquest_api_key: str) -> None: """ Creates a new monitor. Args: origin_lat (float): The origin latitude. origin_lon (float): The origin longitude. mapquest_api_key (str): The MapQuest API key for geocoding. """ self._ses = requests.Session() self._parser = RealTimeParser() self._coder = Geocoder(mapquest_api_key) self._origin = (origin_lat, origin_lon) self._incidents = {} # type: Dict[str, Incident] self._ses.headers.update({ "User-Agent": "SFD Feed Watcher " "(https://github.com/xyx0826/sea_fires_around_me)" }) def _get_two_day_rows(self) -> "List[IncidentRow]": """ Gets all incident rows from today and yesterday to deal with midnight jumps. Returns: List[IncidentRow]: A list of all active incident rows. """ r = self._ses.get(REALTIME_ENDPOINT, params={"action": "Today"}) self._parser.feed(r.text) rows = self._parser.get_rows() yesterday = datetime.today() - timedelta(days=1) r = self._ses.get(REALTIME_ENDPOINT, params={"incDate": yesterday.strftime("%m/%d/%Y")}) self._parser.feed(r.text) yesterday_rows = self._parser.get_rows() if len(yesterday_rows) > 0: rows += yesterday_rows return rows def _add_incident(self, row: IncidentRow) -> None: """ Adds a new incident and prints a message. Args: row (IncidentRow): The new incident. """ latlon = self._coder.geocode(row.loc) inc = Incident(row.id, row.datetime, row.loc, latlon, row.typ) dist = inc.get_dist_to(self._origin) dire = inc.get_direction_to(self._origin) print(f"Incident of type {inc.get_type()} is opened " f"at {inc.get_addr()}, {dist:.2f} km {dire}.") self._incidents[row.id] = inc def _update_incident(self, row: IncidentRow) -> None: """ Updates an open incident. Args: row (IncidentRow): The incident to update. """ inc = self._incidents[row.id] for unit in row.units.split(" "): inc.add_unit(unit) inc.update_type(row.typ) def _remove_incident(self, inc_id: str) -> None: """ Removes a resolved incident and prints a message. Args: inc_id (str): The ID of the resolved incident. """ inc = self._incidents[inc_id] addr = inc.get_addr() mins = inc.get_time_since().total_seconds() / 60 print(f"Incident at {addr} is now resolved after {int(mins)} minutes.") del self._incidents[inc_id] def update(self) -> None: """ Checks if there are incidents to update. """ rows = self._get_two_day_rows() # Add new rows for row in rows: if row.id not in self._incidents.keys(): self._add_incident(row) # Update existing rows updated_rows = [r for r in rows if r.id in self._incidents.keys()] for updated_row in updated_rows: self._update_incident(updated_row) # Remove deleted rows active_ids = [r.id for r in rows] for inc_id in list(self._incidents): if inc_id not in active_ids: self._remove_incident(inc_id)
def main(): #TODO: make this a callback function for verification if options.geocode and not options.geonames_username: logger.error("Please enter a Geonames username to use geocoding") sys.exit(0) directory = options.directory + os.sep output_path = directory + os.sep + "rf2zoo" + os.sep output_path = os.path.normpath(output_path) if not os.path.isdir(output_path): os.mkdir(output_path) logger.info("Writing to %s" % output_path) all_folders = glob.glob(directory + "*") if options.geocode: from geocoder import Geocoder geocoder = Geocoder(options.geonames_username, options.skip_cache) for folder in all_folders: if not os.path.isdir(folder): continue #tODO: allow support for specifying the edge file directly # Check valid format of x:y where x, y are valid ASNs (_, data_folder) = os.path.split(folder) if not re.match("\d+:\d+", data_folder): continue # Only do internal topologies if options.intra_as: (asn_a, asn_b) = data_folder.split(":") if asn_a != asn_b: continue logger.info("Processing %s" % data_folder) G = nx.Graph() #TODO: check on bidirectionality #TODO: download file if not present #TODO: add pyparsing to topzootools dependencies for Pypi # Definitions for parsing colon = Literal(":").suppress() #TODO: see if comma built in comma = Literal(",").suppress() arrow = Literal("->").suppress() ASN = Word(nums) ObsCount = Word(nums) # Can have . eg St. Louis place_name = Group( OneOrMore(Word(alphas+".")) + Optional(comma) + Optional(Word(alphas))) node = Group(ASN + colon + place_name) entry = (node + arrow + node + ObsCount) filename = folder + os.sep + "edges" f = open(filename) #ToDO; print unparsed lines #ToDo: push all into dict and use add_nodes_from and add_edges_from for line in f: #print line processed = entry.parseString(line) #print processed (src, dst, obs_count) = processed src_asn, src_place = src dst_asn, dst_place = dst src_place = " ".join(src_place) dst_place = " ".join(dst_place) # Use simple string for ID (not list returned from Pyparsing which # don't always hash the same src_id = "%s %s" % (src_place, src_asn) dst_id = "%s %s" % (dst_place, dst_asn) # use full hash for node id G.add_node(src_id, ASN=int(src_asn), label =src_place ) G.add_node(dst_id, ASN=int(dst_asn), label=dst_place ) G.add_edge(src_id, dst_id, obs_count = int(obs_count)) f.close() # Relabel nodes to have id based on index if options.geocode: G = geocoder.geocode(G) geocoder.save_cache() G.name = data_folder G.graph['Creator'] = 'rocketfuel2zoo' out_file = output_path + os.sep + data_folder.replace(":", "_") + ".gml" #import pprint #pprint.pprint( sorted(G.nodes())) #pprint.pprint( G.nodes(data=True)) nx.write_gml(G, out_file)
from __future__ import print_function import sys from geocoder import Geocoder if len(sys.argv) < 4: print ("[USAGE]: python",sys.argv[0],"[state abbr filepath] [city filepath] [location text filepath]") exit() state_abbr_filepath = sys.argv[1] city_filepath = sys.argv[2] location_text_filepath = sys.argv[3] gc = Geocoder(state_abbr_filepath, city_filepath) for line in open(location_text_filepath, 'r'): location_text = line.rstrip() point = gc.geocode(location_text) if point == None: print (None) else: print (point[0], point[1])
def main(): network_files = [] if options.file: network_files.append(options.file) if options.directory: #TODO: make this os.path.join across all zootools to remove need for trailing / network_files = glob.glob(options.directory + "*.graphml") if len(network_files) == 0: logger.warn("No files found. Specify -f file or -d directory") sys.exit(0) if options.directory: path = options.directory elif options.file: path = os.path.split(options.file)[0] # Get full path - don't want to create in root dir path = os.path.abspath(path) git_version = get_git_version(path) topzootools_version = pkg_resources.get_distribution("TopZooTools").version if options.output_dir: output_path = options.output_dir else: output_path = path + os.sep + "zoogml" if options.geocode: output_path += "_geocoded" if not os.path.isdir(output_path): os.mkdir(output_path) # clean up path output_path = os.path.normpath(output_path) #TODO: check nx write pickle uses cPickle #TODO: make this optional output_pickle_path = output_path + os.sep + "cache" if not os.path.isdir(output_pickle_path): os.mkdir(output_pickle_path) geocoder = Geocoder(options.geocode, options.skip_cache) #output_path += strftime("%Y%m%d_%H%M%S") logger.info("Saving to folder: %s" % output_path) # and create cache directory for pickle files pickle_dir = path + os.sep + "cache" if not os.path.isdir(pickle_dir): os.mkdir(pickle_dir) #ToDO: check why sorting for source_file in sorted(network_files): # Extract name of network from file path filename = os.path.split(source_file)[1] net_name = os.path.splitext(filename)[0] logger.info( "Converting {0}".format(net_name)) pickle_file = "{0}/{1}.pickle".format(pickle_dir, net_name) if (os.path.isfile(pickle_file) and os.stat(source_file).st_mtime < os.stat(pickle_file).st_mtime): # Pickle file exists, and source_file is older graph = nx.read_gpickle(pickle_file) else: # No pickle file, or is outdated graph = nx.read_graphml(source_file) nx.write_gpickle(graph, pickle_file) graph = convert_to_undirected(graph) # Check for self loops for n, data in graph.nodes(data=True): if n in graph.neighbors(n): logger.warn( "Self loop {0} {1}".format(data['label'], n)) # if all nodes in network are internal, remove redundant internal tag # this makes checking if internal too complex, keep tag for all nodes """ if all(data['Internal'] == 1 for n,data in graph.nodes(data=True)): # all internal, remove tag for n in graph.nodes(): del graph.node[n]['Internal'] """ #********************************* # Geocoding (optional) #********************************* if options.geocode: graph = geocoder.geocode(graph) geocoder.save_cache() #********************************* # Remove graphics data #********************************* # Will be overwritten with name from metadata if present network_label = net_name # extract edge data for src, dst, key, data in graph.edges(data=True, keys=True): if data.get("label"): label = data['label'] extracted_data = extract_edge_data(label) # Replace the edge data with extracted graph.edge[src][dst][key] = extracted_data # remove empty note if 'Note' in graph.graph and graph.graph['Note'] == '': del graph.graph['Note'] # Strip & as yEd fails on it #TODO: use html entitites fn for this network_label_clean = network_label.replace("&", "and") # Set other graph attributes graph.graph['SourceGitVersion'] = git_version graph.graph['Creator'] = "Topology Zoo Toolset" graph.graph['ToolsetVersion'] = topzootools_version graph.graph['label'] = network_label_clean #********************************* #OUTPUT - Write the graphs to files #********************************* filename = network_label.strip() #filename = filename.title() filename = filename.replace(" ", "_") pattern = re.compile('[\W_]+') filename = pattern.sub('', filename) # And also the pickle cache - write first so older timestamp pickle_out_file = "{0}/{1}.pickle".format(output_pickle_path, filename) nx.write_gpickle(graph, pickle_out_file) gml_file = "{0}/{1}.gml".format(output_path, filename) nx.write_gml(graph, gml_file) logger.info("Wrote to %s" % gml_file)