def main():
env = dotenv_values()
db_creds = {
key: env[key]
for key in ["dbname", "user", "password", "host", "port"]
}
conn = db.connect(**db_creds)
iter_conn = db.connect(**db_creds)
gc = Geocoder(conn, iter_conn, debug=False)
gc.geocode()
conn.close()
iter_conn.close()
def get_coordinates(self):
geocoder = Geocoder()
uri = urlparse(self.path)
query = parse_qs(uri.query)
if query.get('address') is None:
return None
coordinates = geocoder.geocode(query['address'][0])
return coordinates
def geocode_city(self, city_state_country):
client = Geocoder("http://open.mapquestapi.com/nominatim/v1/search?format=json")
response = client.geocode(city_state_country)
if (not response):
print ("no response for city=%s" % (city_state_country))
return None
#print (response)
#print response[0][self.DISPLAY_NAME]
#print response[0][self.BOUNDINGBOX]
#print response[0][self.ADDRESS]
'''return boundary of city in format [bottom, top, left, right]'''
return response[0][self.BOUNDINGBOX]
def post(self):
lnd = self.request.get('lnd')
if lnd:
geo = Geocoder(apikey=apikey)
lnd_coord = geo.geocode(location=lnd)
if lnd_coord:
shops = Shop.all()
shops = [
(shop,
calc_distance(shop, lnd_coord),
mapurl(shop.geo.lat, shop.geo.lon)
) for shop in shops
]
shops.sort(key=lambda k: k[1]['distance'])
self.render_response('mobile/search_result.html', {
'lnd': lnd,
'lnd_map': mapurl(lnd_coord['lat'], lnd_coord['lng']),
'shops': shops[0:30]
})
else:
self.response.out.write('not found')
else:
self.redirect('/m/search')
def __new_bot(self):
timezone = TimezoneApi('bar')
timezone.load = MagicMock(return_value=12345)
darksky = WeatherSource('foo', timezone)
darksky.load = MagicMock(
return_value=Weather(now=WeatherAtTime(20, 'Clear', ''),
day=WeatherDay(19, 21, 'Mostly Cloudy', '')))
geocoder = Geocoder('foo')
geocoder.geocode = MagicMock(return_value={
'results': [{
'geometry': {
'location': {
'lat': 1.2,
'lng': 3.4
}
}
}]
})
webcam_source = WebcamSource('foo')
webcam_source.load = MagicMock(return_value=None)
return WeatherBot(darksky, geocoder, webcam_source)
class PostalCodeParser:
def columnNames(self): return [
COUNTRY_CODE,
NAME, # really zipcode
PLACE_NAME,
ADMIN1_NAME,
ADMIN1_CODE,
ADMIN2_NAME,
ADMIN2_CODE,
ADMIN3_NAME,
ADMIN3_CODE,
LATITUDE,
LONGITUDE,
ACCURACY
]
def __init__(self, filename):
self.csvReader = unicode_csv_reader(open(filename), delimiter='\t', quotechar = '\x07')
self.lastRow = self.csvReader.next()
self.count = 0
self.geocoder = Geocoder()
self.done = None
def makeFeature(self, row):
if len(row) != len(self.columnNames()):
print row
print("col length mismatch: %d vs %d" % (len(row), len(self.columnNames())))
values = {}
for (key, val) in zip(self.columnNames(), row):
values[key] = val
return Feature(values)
# make this a real iterator?
def getNextFeatureSet(self):
first_feature = self.makeFeature(self.lastRow)
self.lastRow = None
features = [first_feature]
for row in self.csvReader:
self.count += 1
if self.count % 1000 == 0:
print "imported %d zips so far" % self.count
self.lastRow = row
feature = self.makeFeature(row)
if feature.all_names() == first_feature.all_names():
features.append(feature)
else:
break
if self.lastRow is None:
self.done = True
return features
# try to find each of the admin codes
# if we can't find it, create and return them
def find_and_create_parents(self, feature):
def tooFar(candidate):
km = distance_km(feature.latitude(), feature.longitude(), candidate[0]['lat'], candidate[0]['lng'])
return km < 20000
# try to find CC, then admin1, then admin2, then admin3, then place_name
# if we find it and the name disagrees, add that
# if we can't find it, try geocoding for it instead
cc = feature.country_code()
parents = [feature.cc_id()]
def tryGeocoder(query, record):
parents = []
(geocodes, meta) = self.geocoder.geocode(query)
geocodes = filter(tooFar, geocodes)
if len(meta['query']) == 0 and len(geocodes) > 0:
for g in geocodes:
parents.append(g[0]['featureid'])
if 'parents' in g[0]:
parents += g[0]['parents']
else:
record['parents'] = list(set(filter(None, record['parents'])))
geonames.insert(record)
parents.append(record['featureid'])
return parents
fcodes = [ADM1, ADM2, ADM3]
codes = [ feature.admin1_id(), feature.admin2_id(), feature.admin3_id() ]
names = [ feature.values[ADMIN1_NAME], feature.values[ADMIN2_NAME], feature.values[ADMIN3_NAME] ]
for i in xrange(0, len(codes)):
code = codes[i]
name = names[i]
if not code:
continue
geocodes = [g for g in geonames.find({'featureid': code, 'cc': cc,
'parents': { '$all': filter(None,codes[0:i]) + [feature.cc_id()] }
})]
if len(geocodes):
parents.append(code)
for g in geocodes:
if normalize(name) not in g['names']:
geonames.update({'_id': g['_id']}, {'$addToSet': { 'names': normalize(name) }}, safe=True)
else:
query = ' '.join(names[0:i+1]) + ' ' + cc
parents += tryGeocoder(query,
{
'cc': feature.country_code(),
'feature_code': 'A',
'feature_class': fcodes[i],
'featureid': code,
'names': [normalize(name)],
'lat': feature.latitude(),
'lng': feature.longitude(),
'parents': filter(None, parents),
})
# looking for place name
query = feature.values[PLACE_NAME] + ' ' + ' '.join(names) + ' ' + cc
parents += tryGeocoder(query,
{
'cc': feature.country_code(),
'feature_code': 'P',
'feature_class': 'PPL',
'featureid': normalize(feature.values[PLACE_NAME]),
'names': [normalize(feature.values[PLACE_NAME])],
'lat': feature.latitude(),
'lng': feature.longitude(),
'parents': filter(None, parents),
'displayNames': [{
'l': 'en',
's': feature.name(),
'p': True
}]
})
return list(set(parents))
def parse(self):
while not self.done:
featureSet = self.getNextFeatureSet()
parents = []
last_feature = None
for feature in featureSet:
if last_feature and last_feature.values[PLACE_NAME] == feature.values[PLACE_NAME]:
continue
if feature.country_code() == 'PT':
next
parents = feature.parents()
all_names = feature.all_names()
# with better run-time ranking, we could do this instead of synthesizing parents
# all_names.append(last_feature.values[PLACE_NAME])
if options.create_missing_parents:
parents += self.find_and_create_parents(feature)
last_feature = feature
# save the first feature
feature = featureSet[0]
try:
record = {
'featureid': '%s-%s' % (feature.country_code(), all_names[0]),
'cc': feature.country_code(),
'feature_code': 'Z',
'feature_class': 'ZIP',
'names': [normalize(n) for n in all_names],
'lat': feature.latitude(),
'lng': feature.longitude(),
'parents': list(set(parents)),
'accuracy': feature.accuracy(),
'displayNames': [{
'l': 'en',
's': feature.name(),
'p': True
}]
}
except:
error(values, row)
geonames.insert(record)
import sys
from geocoder import Geocoder
if len(sys.argv) < 4:
print "[USAGE]: python %s [state abbr filepath] [city filepath] [location text filepath]" % sys.argv[0]
exit()
state_abbr_filepath = sys.argv[1]
city_filepath = sys.argv[2]
location_text_filepath = sys.argv[3]
gc = Geocoder(state_abbr_filepath, city_filepath)
for line in open(location_text_filepath, 'r'):
location_text = line.rstrip()
point = gc.geocode(location_text)
if point == None:
print None
else:
print point[0], point[1]
query = u"%s %s %s" % (
f['properties']['NAME'],
f['properties']['ADM1NAME'],
f['properties']['ADM0NAME']
)
print query
ll = (f['properties']['LATITUDE'], f['properties']['LONGITUDE'])
llStr = '%s,%s' % ll
hint = {
'woeHint': 'TOWN',
'll': llStr
}
try:
geocode = geocoder.geocode(query, hint)
if not geocode:
query = u"%s %s" % (
f['properties']['ADM1NAME'],
f['properties']['ADM0NAME']
)
geocode = geocoder.geocode(query, hint)
except:
next
f['properties']['db_geonameid'] = ''
f['properties']['db_lat'] = ''
f['properties']['db_lng'] = ''
f['properties']['db_distance'] = 0.0
class RealTime:
"""
A real-time 911 dispatch monitor.
"""
def __init__(self, origin_lat: float, origin_lon: float,
mapquest_api_key: str) -> None:
"""
Creates a new monitor.
Args:
origin_lat (float): The origin latitude.
origin_lon (float): The origin longitude.
mapquest_api_key (str): The MapQuest API key for geocoding.
"""
self._ses = requests.Session()
self._parser = RealTimeParser()
self._coder = Geocoder(mapquest_api_key)
self._origin = (origin_lat, origin_lon)
self._incidents = {} # type: Dict[str, Incident]
self._ses.headers.update({
"User-Agent":
"SFD Feed Watcher "
"(https://github.com/xyx0826/sea_fires_around_me)"
})
def _get_two_day_rows(self) -> "List[IncidentRow]":
"""
Gets all incident rows from today and yesterday
to deal with midnight jumps.
Returns:
List[IncidentRow]: A list of all active incident rows.
"""
r = self._ses.get(REALTIME_ENDPOINT, params={"action": "Today"})
self._parser.feed(r.text)
rows = self._parser.get_rows()
yesterday = datetime.today() - timedelta(days=1)
r = self._ses.get(REALTIME_ENDPOINT,
params={"incDate": yesterday.strftime("%m/%d/%Y")})
self._parser.feed(r.text)
yesterday_rows = self._parser.get_rows()
if len(yesterday_rows) > 0:
rows += yesterday_rows
return rows
def _add_incident(self, row: IncidentRow) -> None:
"""
Adds a new incident and prints a message.
Args:
row (IncidentRow): The new incident.
"""
latlon = self._coder.geocode(row.loc)
inc = Incident(row.id, row.datetime, row.loc, latlon, row.typ)
dist = inc.get_dist_to(self._origin)
dire = inc.get_direction_to(self._origin)
print(f"Incident of type {inc.get_type()} is opened "
f"at {inc.get_addr()}, {dist:.2f} km {dire}.")
self._incidents[row.id] = inc
def _update_incident(self, row: IncidentRow) -> None:
"""
Updates an open incident.
Args:
row (IncidentRow): The incident to update.
"""
inc = self._incidents[row.id]
for unit in row.units.split(" "):
inc.add_unit(unit)
inc.update_type(row.typ)
def _remove_incident(self, inc_id: str) -> None:
"""
Removes a resolved incident and prints a message.
Args:
inc_id (str): The ID of the resolved incident.
"""
inc = self._incidents[inc_id]
addr = inc.get_addr()
mins = inc.get_time_since().total_seconds() / 60
print(f"Incident at {addr} is now resolved after {int(mins)} minutes.")
del self._incidents[inc_id]
def update(self) -> None:
"""
Checks if there are incidents to update.
"""
rows = self._get_two_day_rows()
# Add new rows
for row in rows:
if row.id not in self._incidents.keys():
self._add_incident(row)
# Update existing rows
updated_rows = [r for r in rows if r.id in self._incidents.keys()]
for updated_row in updated_rows:
self._update_incident(updated_row)
# Remove deleted rows
active_ids = [r.id for r in rows]
for inc_id in list(self._incidents):
if inc_id not in active_ids:
self._remove_incident(inc_id)
def main():
#TODO: make this a callback function for verification
if options.geocode and not options.geonames_username:
logger.error("Please enter a Geonames username to use geocoding")
sys.exit(0)
directory = options.directory + os.sep
output_path = directory + os.sep + "rf2zoo" + os.sep
output_path = os.path.normpath(output_path)
if not os.path.isdir(output_path):
os.mkdir(output_path)
logger.info("Writing to %s" % output_path)
all_folders = glob.glob(directory + "*")
if options.geocode:
from geocoder import Geocoder
geocoder = Geocoder(options.geonames_username, options.skip_cache)
for folder in all_folders:
if not os.path.isdir(folder):
continue
#tODO: allow support for specifying the edge file directly
# Check valid format of x:y where x, y are valid ASNs
(_, data_folder) = os.path.split(folder)
if not re.match("\d+:\d+", data_folder):
continue
# Only do internal topologies
if options.intra_as:
(asn_a, asn_b) = data_folder.split(":")
if asn_a != asn_b:
continue
logger.info("Processing %s" % data_folder)
G = nx.Graph()
#TODO: check on bidirectionality
#TODO: download file if not present
#TODO: add pyparsing to topzootools dependencies for Pypi
# Definitions for parsing
colon = Literal(":").suppress()
#TODO: see if comma built in
comma = Literal(",").suppress()
arrow = Literal("->").suppress()
ASN = Word(nums)
ObsCount = Word(nums)
# Can have . eg St. Louis
place_name = Group(
OneOrMore(Word(alphas+"."))
+ Optional(comma) +
Optional(Word(alphas)))
node = Group(ASN + colon + place_name)
entry = (node + arrow + node + ObsCount)
filename = folder + os.sep + "edges"
f = open(filename)
#ToDO; print unparsed lines
#ToDo: push all into dict and use add_nodes_from and add_edges_from
for line in f:
#print line
processed = entry.parseString(line)
#print processed
(src, dst, obs_count) = processed
src_asn, src_place = src
dst_asn, dst_place = dst
src_place = " ".join(src_place)
dst_place = " ".join(dst_place)
# Use simple string for ID (not list returned from Pyparsing which
# don't always hash the same
src_id = "%s %s" % (src_place, src_asn)
dst_id = "%s %s" % (dst_place, dst_asn)
# use full hash for node id
G.add_node(src_id, ASN=int(src_asn), label =src_place )
G.add_node(dst_id, ASN=int(dst_asn), label=dst_place )
G.add_edge(src_id, dst_id, obs_count = int(obs_count))
f.close()
# Relabel nodes to have id based on index
if options.geocode:
G = geocoder.geocode(G)
geocoder.save_cache()
G.name = data_folder
G.graph['Creator'] = 'rocketfuel2zoo'
out_file = output_path + os.sep + data_folder.replace(":", "_") + ".gml"
#import pprint
#pprint.pprint( sorted(G.nodes()))
#pprint.pprint( G.nodes(data=True))
nx.write_gml(G, out_file)
from __future__ import print_function
import sys
from geocoder import Geocoder
if len(sys.argv) < 4:
print ("[USAGE]: python",sys.argv[0],"[state abbr filepath] [city filepath] [location text filepath]")
exit()
state_abbr_filepath = sys.argv[1]
city_filepath = sys.argv[2]
location_text_filepath = sys.argv[3]
gc = Geocoder(state_abbr_filepath, city_filepath)
for line in open(location_text_filepath, 'r'):
location_text = line.rstrip()
point = gc.geocode(location_text)
if point == None:
print (None)
else:
print (point[0], point[1])
def main():
network_files = []
if options.file:
network_files.append(options.file)
if options.directory:
#TODO: make this os.path.join across all zootools to remove need for trailing /
network_files = glob.glob(options.directory + "*.graphml")
if len(network_files) == 0:
logger.warn("No files found. Specify -f file or -d directory")
sys.exit(0)
if options.directory:
path = options.directory
elif options.file:
path = os.path.split(options.file)[0]
# Get full path - don't want to create in root dir
path = os.path.abspath(path)
git_version = get_git_version(path)
topzootools_version = pkg_resources.get_distribution("TopZooTools").version
if options.output_dir:
output_path = options.output_dir
else:
output_path = path + os.sep + "zoogml"
if options.geocode:
output_path += "_geocoded"
if not os.path.isdir(output_path):
os.mkdir(output_path)
# clean up path
output_path = os.path.normpath(output_path)
#TODO: check nx write pickle uses cPickle
#TODO: make this optional
output_pickle_path = output_path + os.sep + "cache"
if not os.path.isdir(output_pickle_path):
os.mkdir(output_pickle_path)
geocoder = Geocoder(options.geocode, options.skip_cache)
#output_path += strftime("%Y%m%d_%H%M%S")
logger.info("Saving to folder: %s" % output_path)
# and create cache directory for pickle files
pickle_dir = path + os.sep + "cache"
if not os.path.isdir(pickle_dir):
os.mkdir(pickle_dir)
#ToDO: check why sorting
for source_file in sorted(network_files):
# Extract name of network from file path
filename = os.path.split(source_file)[1]
net_name = os.path.splitext(filename)[0]
logger.info( "Converting {0}".format(net_name))
pickle_file = "{0}/{1}.pickle".format(pickle_dir, net_name)
if (os.path.isfile(pickle_file) and
os.stat(source_file).st_mtime < os.stat(pickle_file).st_mtime):
# Pickle file exists, and source_file is older
graph = nx.read_gpickle(pickle_file)
else:
# No pickle file, or is outdated
graph = nx.read_graphml(source_file)
nx.write_gpickle(graph, pickle_file)
graph = convert_to_undirected(graph)
# Check for self loops
for n, data in graph.nodes(data=True):
if n in graph.neighbors(n):
logger.warn( "Self loop {0} {1}".format(data['label'], n))
# if all nodes in network are internal, remove redundant internal tag
# this makes checking if internal too complex, keep tag for all nodes
"""
if all(data['Internal'] == 1 for n,data in graph.nodes(data=True)):
# all internal, remove tag
for n in graph.nodes():
del graph.node[n]['Internal']
"""
#*********************************
# Geocoding (optional)
#*********************************
if options.geocode:
graph = geocoder.geocode(graph)
geocoder.save_cache()
#*********************************
# Remove graphics data
#*********************************
# Will be overwritten with name from metadata if present
network_label = net_name
# extract edge data
for src, dst, key, data in graph.edges(data=True, keys=True):
if data.get("label"):
label = data['label']
extracted_data = extract_edge_data(label)
# Replace the edge data with extracted
graph.edge[src][dst][key] = extracted_data
# remove empty note
if 'Note' in graph.graph and graph.graph['Note'] == '':
del graph.graph['Note']
# Strip & as yEd fails on it
#TODO: use html entitites fn for this
network_label_clean = network_label.replace("&", "and")
# Set other graph attributes
graph.graph['SourceGitVersion'] = git_version
graph.graph['Creator'] = "Topology Zoo Toolset"
graph.graph['ToolsetVersion'] = topzootools_version
graph.graph['label'] = network_label_clean
#*********************************
#OUTPUT - Write the graphs to files
#*********************************
filename = network_label.strip()
#filename = filename.title()
filename = filename.replace(" ", "_")
pattern = re.compile('[\W_]+')
filename = pattern.sub('', filename)
# And also the pickle cache - write first so older timestamp
pickle_out_file = "{0}/{1}.pickle".format(output_pickle_path, filename)
nx.write_gpickle(graph, pickle_out_file)
gml_file = "{0}/{1}.gml".format(output_path, filename)
nx.write_gml(graph, gml_file)
logger.info("Wrote to %s" % gml_file)
