def create(cls, **kwargs):
"""Insert a new Muni stop entry with all corresponding geoboxes.
Args:
system: The muni system to use. Always "sf-muni" right now.
lat, lon: Coordinates of stop.
stop_id: The stop ID of the stop.
in_routes: List of inbound routes.
out_routes: List of outbound routes.
"""
all_boxes = []
lat = kwargs.pop("lat")
lon = kwargs.pop("lon")
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
kwargs["stop_id"] = str(kwargs["stop_id"])
kwargs["location"] = db.GeoPt(lat, lon)
kwargs["key_name"] = "stop:%s:%s" % (kwargs["system"], kwargs["stop_id"])
kwargs["geoboxes"] = all_boxes
kwargs["has_inbound_routes"] = bool(len(kwargs["in_routes"]) > 0)
kwargs["has_outbound_routes"] = bool(len(kwargs["out_routes"]) > 0)
return cls(**kwargs)
def create(cls, **kwargs):
"""Insert a new Muni stop entry with all corresponding geoboxes.
Args:
system: The muni system to use. Always "sf-muni" right now.
lat, lon: Coordinates of stop.
stop_id: The stop ID of the stop.
in_routes: List of inbound routes.
out_routes: List of outbound routes.
"""
all_boxes = []
lat = kwargs.pop("lat")
lon = kwargs.pop("lon")
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(
geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
kwargs["stop_id"] = str(kwargs["stop_id"])
kwargs["location"] = db.GeoPt(lat, lon)
kwargs["key_name"] = "stop:%s:%s" % (kwargs["system"],
kwargs["stop_id"])
kwargs["title"] = route_data.STOP_BY_ID[int(kwargs["stop_id"])][0]
kwargs["geoboxes"] = all_boxes
kwargs["has_inbound_routes"] = bool(len(kwargs["in_routes"]) > 0)
kwargs["has_outbound_routes"] = bool(len(kwargs["out_routes"]) > 0)
return cls(**kwargs)
def near_location(self, lat, lon, max_results, min_params):
found_momentos = {}
# Do concentric queries until the max number of results is reached.
for params in GEOBOX_CONFIGS:
if len(found_momentos) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s", box, resolution, slice)
query = Momento.query(Momento.geoboxes == box)
results = query.fetch(100)
logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.key not in found_momentos:
found_momentos[result.key] = result
# Now compute distances and sort by distance.
momentos_by_distance = []
for momento in found_momentos.itervalues():
distance = _earth_distance(lat, lon, momento.location.lat, momento.location.lon)
momentos_by_distance.append((distance, momento))
momentos_by_distance.sort()
# return momentos_by_distance
return momentos_by_distance
def query(cls, lat, lon, max_results, min_params):
found_resorts = {}
# Do concentric queries until the max number of results is reached.
# Use only the first three geoboxes for search to reduce query overhead.
for params in GEOBOX_CONFIGS[:3]:
if len(found_resorts) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
results = query.fetch(50)
logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.name not in found_resorts:
found_resorts[result.name] = result
# Now compute distances and sort by distance.
resorts_by_distance = []
for resort in found_resorts.itervalues():
distance = distance_on_earth(lat, lon, resort.location.lat, resort.location.lon)
resorts_by_distance.append((distance, resort))
resorts_by_distance.sort()
return resorts_by_distance[:max_results]
def query(cls, lat, lon, max_results, min_params):
found_resorts = {}
# Do concentric queries until the max number of results is reached.
# Use only the first three geoboxes for search to reduce query overhead.
for params in GEOBOX_CONFIGS[:3]:
if len(found_resorts) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
results = query.fetch(50)
logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.name not in found_resorts:
found_resorts[result.name] = result
# Now compute distances and sort by distance.
resorts_by_distance = []
for resort in found_resorts.itervalues():
distance = distance_on_earth(lat, lon, resort.location.lat,
resort.location.lon)
resorts_by_distance.append((distance, resort))
resorts_by_distance.sort()
return resorts_by_distance[:max_results]
def query(cls, max_results=1, min_params=(0,2), time_limit = TIME_TO_EXPIRE,**kwargs):
lat = kwargs.pop("lat")
lon = kwargs.pop("lon")
id = kwargs.pop("id")
user = getUser(id)
if not user:
raise Exception("User not found in database")
userLocation = db.GqlQuery("SELECT * FROM UserLocation WHERE id=:1",id).get()
if not userLocation:
logging.info("Querying before updating! A big no-no.")
return
logging.info("Looking for friends nearby for %s (has %s as friends)" % (user.name, str(user.friendsPlaying)))
found_friends = {}
for params in GEOBOX_CONFIGS[:3]:
if len(found_friends) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
query.filter("id in", user.friendsPlaying)
tmp = cls.all()
tmp.filter("isBiz =", True)
results2 = tmp.fetch(50)
logging.info("Businesses nearby: %d", len(results2))
results = query.fetch(50)
results.extend(results2)
logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.id not in found_friends:
found_friends[result.id] = result
# Now compute distances and sort by distance.
friends_by_distance = []
for friend in found_friends.itervalues():
ut = userLocation.time
ft = friend.time
date_diff = ut - ft if ut > ft else ft - ut
if date_diff.days == 0 and date_diff.seconds < time_limit:
distance = _earth_distance(lat, lon, friend.location.lat, friend.location.lon)
friends_by_distance.append((distance, friend))
else:
logging.info("Didn't return result for user %s (time limit: %s)" % (friend.id, str(date_diff)))
friends_by_distance.sort()
return friends_by_distance[:max_results]
def process(lat, lon):
all_boxes = []
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
return all_boxes, db.GeoPt(lat, lon)
def query_location(cls, query=None, lat=None, lon=None):
"""
Returns a geolocation query that can be further filtered
"""
if not query:
query = Business.query()
box = geobox.compute(lat, lon, 1, 1)
query = query.filter(Business.geoboxes == box)
return query
def query(cls, system, lat, lon, inbound, max_results, min_params):
"""Queries for Muni stops repeatedly until max results or scope is reached.
Args:
system: The transit system to query.
lat, lon: Coordinates of the agent querying.
inbound: If the routes we're interested in are inbound.
max_results: Maximum number of stops to find.
min_params: Tuple (resolution, slice) of the minimum resolution to allow.
Returns:
List of (distance, MuniStop) tuples, ordered by minimum distance first.
There will be no duplicates in these results. Distance is in meters.
"""
# Maps stop_ids to MuniStop instances.
found_stops = {}
# Do concentric queries until the max number of results is reached.
# Use only the first three geoboxes for search to reduce query overhead.
for params in GEOBOX_CONFIGS[:3]:
if len(found_stops) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
query.filter("system =", system)
if inbound:
query.filter("has_inbound_routes =", True)
else:
query.filter("has_outbound_routes =", True)
results = query.fetch(50)
logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.stop_id not in found_stops:
found_stops[result.stop_id] = result
# Now compute distances and sort by distance.
stops_by_distance = []
for stop in found_stops.itervalues():
distance = _earth_distance(lat, lon, stop.location.lat,
stop.location.lon)
stops_by_distance.append((distance, stop))
stops_by_distance.sort()
return stops_by_distance[:max_results]
def query(cls, system, lat, lon, inbound, max_results, min_params):
"""Queries for Muni stops repeatedly until max results or scope is reached.
Args:
system: The transit system to query.
lat, lon: Coordinates of the agent querying.
inbound: If the routes we're interested in are inbound.
max_results: Maximum number of stops to find.
min_params: Tuple (resolution, slice) of the minimum resolution to allow.
Returns:
List of (distance, MuniStop) tuples, ordered by minimum distance first.
There will be no duplicates in these results. Distance is in meters.
"""
# Maps stop_ids to MuniStop instances.
found_stops = {}
# Do concentric queries until the max number of results is reached.
# Use only the first three geoboxes for search to reduce query overhead.
for params in GEOBOX_CONFIGS[:3]:
if len(found_stops) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
query.filter("system =", system)
if inbound:
query.filter("has_inbound_routes =", True)
else:
query.filter("has_outbound_routes =", True)
results = query.fetch(50)
logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.stop_id not in found_stops:
found_stops[result.stop_id] = result
# Now compute distances and sort by distance.
stops_by_distance = []
for stop in found_stops.itervalues():
distance = _earth_distance(lat, lon, stop.location.lat, stop.location.lon)
stops_by_distance.append((distance, stop))
stops_by_distance.sort()
return stops_by_distance[:max_results]
def kml_to_csv(neighborhoods, district, data, writer, min_timestamp):
parsetree = minidom.parseString(data)
placemarks = parsetree.getElementsByTagName("Placemark")
for place in placemarks:
try:
description = place.getElementsByTagName('description')
extended_data = {}
all_info = {}
extended_data_nodes = place.getElementsByTagName('SimpleData')
for entry in extended_data_nodes:
entry_name = entry.getAttribute("name")
entry_val = entry.firstChild
extended_data[entry_name.lower()] = entry_val.data
date = time.strptime(extended_data["date"] + ' ' + extended_data["time"], "%m/%d/%Y %H:%M")
if time.mktime(date) < min_timestamp:
continue
for entry_name in ['category', 'description', 'incident', 'resolution']:
all_info[entry_name] = extended_data[entry_name]
# print date
all_info["timestamp"] = time.mktime(date)
location_node = place.getElementsByTagName('Point')[0].getElementsByTagName('coordinates')[0].firstChild.data
lon = float(location_node.split(',')[0])
lat = float(location_node.split(',')[1])
all_info["lon"] = lon
all_info["lat"] = lat
all_info["district"] = district
all_boxes = []
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
all_info["geoboxes"] = all_boxes
all_info["neighborhood"] = find_neighborhood(Point(lat,lon))
writer.writerow(all_info)
except Exception, e:
print e
def query(self, lat, lon, max_results, min_params):
"""Queries for Muni stops repeatedly until max results or scope is reached.
Args:
system: The transit system to query.
lat, lon: Coordinates of the agent querying.
max_results: Maximum number of stops to find.
min_params: Tuple (resolution, slice) of the minimum resolution to allow.
Returns:
List of (distance, MuniStop) tuples, ordered by minimum distance first.
There will be no duplicates in these results. Distance is in meters.
"""
# Maps stop_ids to MuniStop instances.
found_spots = {}
# Do concentric queries until the max number of results is reached.
#dow_query_string = _DAY_DICTIONARY[dow] + ' ='
for params in GEOBOX_CONFIGS:
if len(found_spots) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
#logging.info("Searching for box=%s at resolution=%s, slice=%s",
#box, resolution, slice)
query = self.all().filter("geoboxes =", box)
#results spot objs
results = query.fetch(50)
#logging.info("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.name not in found_spots:
found_spots[result.name] = result
# Now compute distances and sort by distance.
spots_by_distance = []
for spot in found_spots.itervalues():
distance = _earth_distance(lat, lon, spot.location.lat, spot.location.lon)
spots_by_distance.append((distance, spot))
spots_by_distance.sort()
return spots_by_distance
def create(cls, **kwargs):
lat = kwargs['lat']
lon = kwargs['lon']
location = db.GeoPt(lat, lon)
name = kwargs['name']
url = kwargs['url']
area_name = kwargs['area_name']
country = kwargs['country']
all_boxes = []
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
new_resort = Resort(name=name, location=location, lat=lat,
lon=lon, area_name=area_name, country=country,
url=url, geoboxes=all_boxes)
return new_resort
def query(self, time, dow, lat, lon, max_results, min_params):
"""Queries for Muni stops repeatedly until max results or scope is reached.
Args:
system: The transit system to query.
lat, lon: Coordinates of the agent querying.
max_results: Maximum number of stops to find.
min_params: Tuple (resolution, slice) of the minimum resolution to allow.
Returns:
List of (distance, MuniStop) tuples, ordered by minimum distance first.
There will be no duplicates in these results. Distance is in meters.
"""
# Maps stop_ids to MuniStop instances.
found_stores = {}
# Do concentric queries until the max number of results is reached.
dow_query_string = _DAY_DICTIONARY[dow] + ' ='
for params in GEOBOX_CONFIGS:
if len(found_stores) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.info("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = self.all().filter("geoboxes =", box).filter(dow_query_string, time)
results = query.fetch(50)
logging.info("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.name not in found_stores:
found_stores[result.name] = result
# Now compute distances and sort by distance.
stores_by_distance = []
for store in found_stores.itervalues():
distance = _earth_distance(lat, lon, store.location.lat, store.location.lon)
stores_by_distance.append((distance, store))
stores_by_distance.sort()
return stores_by_distance
def add(self, **kwargs):
lat = kwargs.pop('lat')
lon = kwargs.pop('lon')
location = db.GeoPt(lat, lon)
name = kwargs['name']
new_spot = Spot(name=name, location=location)
all_boxes = []
#new_spot.pretty_address = kwargs['address']
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
new_spot.geoboxes = all_boxes
new_spot.description = kwargs['description']
print new_spot.geoboxes
print new_spot.description
new_spot.put()
def add(self, author, text, lat, lon, image):
location = ndb.GeoPt(lat, lon)
momento = Momento(author=author, text=text, location=location)
if image:
momento.image = db.Blob(image)
thumbnail = images.resize(image, 100, 100)
momento.thumbnail = db.Blob(thumbnail)
all_boxes = []
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
# logging.debug("Geoboxes " + str(all_boxes))
momento.geoboxes = all_boxes
momento.put()
def add(self, **kwargs):
lat = kwargs.pop('lat')
lon = kwargs.pop('lon')
location = db.GeoPt(lat, lon)
name = kwargs['name']
new_store = Store(name=name, location=location)
all_boxes = []
new_store.pretty_address = kwargs['address']
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
new_store.geoboxes = all_boxes
store_hour_dict = _make_hours(kwargs['store_hours'])
for day, prop in _DAY_DICTIONARY.iteritems():
setattr(new_store, prop, store_hour_dict[day])
new_store.categories = kwargs['categories']
new_store.pretty_description = kwargs['description']
new_store.put()
def add(self, **kwargs):
lat = kwargs.pop('lat')
lon = kwargs.pop('lon')
location = db.GeoPt(lat, lon)
name = kwargs['name']
new_store = Store(name=name, location=location)
all_boxes = []
new_store.pretty_address = kwargs['address']
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
new_store.geoboxes = all_boxes
store_hour_dict = _make_hours(kwargs['store_hours'])
for day, prop in _DAY_DICTIONARY.iteritems():
setattr(new_store, prop, store_hour_dict[day])
new_store.categories = kwargs['categories']
new_store.pretty_description = kwargs['description']
new_store.put()
def query(cls, lat, lon, max_results, min_params):
"""Queries for incidents repeatedly until max results or scope is reached.
Args:
lat, lon: Coordinates of the agent querying.
max_results: Maximum number of stops to find.
min_params: Tuple (resolution, slice) of the minimum resolution to allow.
Returns:
List of (distance, CrimeEntry) tuples, ordered by minimum distance first.
There will be no duplicates in these results. Distance is in meters.
"""
# Maps stop_ids to CrimeEntry instances.
found_incidents = {}
# Do concentric queries until the max number of results is reached.
# Use only the first three geoboxes for search to reduce query overhead.
for params in GEOBOX_CONFIGS[:3]:
if len(found_incidents) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
results = query.fetch(50)
# logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.incident not in found_incidents:
found_incidents[result.incident] = result
# Now compute distances and sort by distance.
return cls._incidents_by_distance(lat, lon, found_incidents)
def query(cls, lat, lon, max_results, min_params):
"""Queries for incidents repeatedly until max results or scope is reached.
Args:
lat, lon: Coordinates of the agent querying.
max_results: Maximum number of stops to find.
min_params: Tuple (resolution, slice) of the minimum resolution to allow.
Returns:
List of (distance, CrimeEntry) tuples, ordered by minimum distance first.
There will be no duplicates in these results. Distance is in meters.
"""
# Maps stop_ids to CrimeEntry instances.
found_incidents = {}
# Do concentric queries until the max number of results is reached.
# Use only the first three geoboxes for search to reduce query overhead.
for params in GEOBOX_CONFIGS[:3]:
if len(found_incidents) >= max_results:
break
if params < min_params:
break
resolution, slice, unused = params
box = geobox.compute(lat, lon, resolution, slice)
logging.debug("Searching for box=%s at resolution=%s, slice=%s",
box, resolution, slice)
query = cls.all()
query.filter("geoboxes =", box)
results = query.fetch(50)
# logging.debug("Found %d results", len(results))
# De-dupe results.
for result in results:
if result.incident not in found_incidents:
found_incidents[result.incident] = result
# Now compute distances and sort by distance.
return cls._incidents_by_distance(lat, lon, found_incidents)
def create(cls, **kwargs):
lat = kwargs['lat']
lon = kwargs['lon']
location = db.GeoPt(lat, lon)
name = kwargs['name']
url = kwargs['url']
area_name = kwargs['area_name']
country = kwargs['country']
all_boxes = []
for (resolution, slice, use_set) in GEOBOX_CONFIGS:
if use_set:
all_boxes.extend(
geobox.compute_set(lat, lon, resolution, slice))
else:
all_boxes.append(geobox.compute(lat, lon, resolution, slice))
new_resort = Resort(name=name,
location=location,
lat=lat,
lon=lon,
area_name=area_name,
country=country,
url=url,
geoboxes=all_boxes)
return new_resort
def _pre_put_hook(self):
self.geoboxes = [geobox.compute(self.lat, self.lon, 1, 1)]