def get_user_data(self,
user=None,
lat_north=90,
lng_west=-180,
range_lat=-180,
range_lng=360,
max_results=2000):
# log.info("GeoRange: (%6.4f, %6.4f) ZoomStep: (%6.4f, %6.4f)" % (lat_north, lng_west, range_lat, range_lng))
# log.info("Range: (%6.4f - %6.4f), (%6.4f - %6.4f)" % (min(90, max(-90, lat_north + range_lat)), lat_north, min(180, max(-180, lng_west + range_lng)), lng_west))
if user:
# not sure why Google was giving latitudes outside of the allowable range near the International Date Line at zoom level 3,
# but cap it to the max anyway here. this might result in incorrectly drawn tiles near there, but oh well.
if lng_west < -180:
lng_west = -180
return UserVenue.bounding_box_fetch(
UserVenue.all().filter('user ='******'-last_checkin_at'
), #TODO find a way to specify this elsewhere!!
geotypes.Box(min(90, max(-90, lat_north + range_lat)),
min(180, max(-180, lng_west + range_lng)),
lat_north, lng_west),
max_results,
)
else:
return []
def get_all_data(self, user=None, lat_north=90, lng_west=-180, range_lat=-180, range_lng=360, max_results=2000):
if user:
self.get_user_data(user, lat_north, lng_west, range_lat, range_lng, max_results)
else:
return UserVenue.bounding_box_fetch(UserVenue.all().order('-last_checkin_at'),
geotypes.Box(min(90, max(-90, lat_north + range_lat)),
min(180, max(-180, lng_west + range_lng)),
lat_north,
lng_west),
max_results, )
def get_user_data(self, user=None, lat_north=90, lng_west=-180, range_lat=-180, range_lng=360, max_results=1000):
# log.info("GeoRange: (%6.4f, %6.4f) ZoomStep: (%6.4f, %6.4f)" % (lat_north, lng_west, range_lat, range_lng))
# log.info("Range: (%6.4f - %6.4f), (%6.4f - %6.4f)" % (min(90, max(-90, lat_north + range_lat)), lat_north, min(180, max(-180, lng_west + range_lng)), lng_west))
if user: #WARNING last_updated is confusing and should be last_checkin_at
return UserVenue.bounding_box_fetch(UserVenue.all().filter('user ='******'-last_updated'), #TODO find a way to specify this elsewhere!!
geotypes.Box(min(90, max(-90, lat_north + range_lat)),
min(180, max(-180, lng_west + range_lng)),
lat_north,
lng_west),
max_results, )
else:
return []
def get_user_data(self, user=None, lat_north=90, lng_west=-180, range_lat=-180, range_lng=360, max_results=2000):
# log.info("GeoRange: (%6.4f, %6.4f) ZoomStep: (%6.4f, %6.4f)" % (lat_north, lng_west, range_lat, range_lng))
# log.info("Range: (%6.4f - %6.4f), (%6.4f - %6.4f)" % (min(90, max(-90, lat_north + range_lat)), lat_north, min(180, max(-180, lng_west + range_lng)), lng_west))
if user:
# not sure why Google was giving latitudes outside of the allowable range near the International Date Line at zoom level 3,
# but cap it to the max anyway here. this might result in incorrectly drawn tiles near there, but oh well.
if lng_west < -180:
lng_west = -180
return UserVenue.bounding_box_fetch(UserVenue.all().filter('user ='******'-last_checkin_at'), #TODO find a way to specify this elsewhere!!
geotypes.Box(min(90, max(-90, lat_north + range_lat)),
min(180, max(-180, lng_west + range_lng)),
lat_north,
lng_west),
max_results, )
else:
return []
def get_all_data(self,
user=None,
lat_north=90,
lng_west=-180,
range_lat=-180,
range_lng=360,
max_results=2000):
if user:
self.get_user_data(user, lat_north, lng_west, range_lat, range_lng,
max_results)
else:
return UserVenue.bounding_box_fetch(
UserVenue.all().order('-last_checkin_at'),
geotypes.Box(min(90, max(-90, lat_north + range_lat)),
min(180, max(-180, lng_west + range_lng)),
lat_north, lng_west),
max_results,
)
def clear_old_uservenues():
num_cleared = 0
cutoff = datetime.now() - timedelta(days=7)
userinfos = UserInfo.all().filter('has_been_cleared = ', False).filter('last_updated <', cutoff).fetch(200)
try:
for userinfo in userinfos:
while True:
uservenues = UserVenue.all(keys_only=True).filter('user =', userinfo.user).fetch(1000)
if not uservenues: break
db.delete(uservenues)
num_cleared = num_cleared + len(uservenues)
userinfo.has_been_cleared = True
userinfo.checkin_count = 0
userinfo.venue_count = 0
userinfo.last_updated = datetime.now()
userinfo.put()
logging.info("finished after deleting at least %d UserVenues for %d UserInfos" % (num_cleared, len(userinfos)))
except DeadlineExceededError:
logging.info("exceeded deadline after deleting at least %d UserVenues for %d UserInfos" % (num_cleared, len(userinfos)))
def clear_old_uservenues():
num_cleared = 0
cutoff = datetime.now() - timedelta(days=7)
userinfos = UserInfo.all().filter('has_been_cleared = ', False).filter('last_updated <', cutoff).fetch(200)
try:
for userinfo in userinfos:
while True:
uservenues = UserVenue.all(keys_only=True).filter('user =', userinfo.user).fetch(1000)
if not uservenues: break
db.delete(uservenues)
num_cleared = num_cleared + len(uservenues)
userinfo.has_been_cleared = True
userinfo.checkin_count = 0
userinfo.venue_count = 0
userinfo.last_updated = datetime.now()
userinfo.put()
logging.info("finished after deleting at least %d UserVenues for %d UserInfos" % (num_cleared, len(userinfos)))
except DeadlineExceededError:
logging.info("exceeded deadline after deleting at least %d UserVenues for %d UserInfos" % (num_cleared, len(userinfos)))
# db.delete(mapimages)
# while(UserVenue.all().count() > 0):
# uservenues = UserVenue.all().fetch(500)
# db.delete(uservenues)
# while(AuthToken.all().count() > 0):
# authtokens = AuthToken.all().fetch(500)
# db.delete(authtokens)
# while(UserInfo.all().count() > 0):
# userinfos = UserInfo.all().fetch(500)
# db.delete(userinfos)
elif rest == 'user':
user = users.get_current_user()
if user:
while (MapImage.all().filter('user ='******'user ='******'user = '******'user = '******'user ='******'user ='******'mapimage':
user = users.get_current_user()
if user:
while (MapImage.all().filter('user ='******'user =', user).fetch(500)
db.delete(mapimages)
def uservenue_factory(userinfo_param, j_venue_param, checkin_guid_list_param, checkin_list_param, is_unique_param):
new_uservenue = UserVenue(parent=userinfo_param, location = db.GeoPt(j_venue_param['geolat'], j_venue_param['geolong']))
new_uservenue.update_location()
new_uservenue.user = userinfo_param.user
new_uservenue.venue_guid = str(j_venue_param['id'])
if 'name' in j_venue_param:
new_uservenue.name = j_venue_param['name']
if 'address' in j_venue_param:
new_uservenue.address = j_venue_param['address'].replace('\n', ' ').replace('\r', ' ')
if 'cross_street' in j_venue_param:
new_uservenue.cross_street = j_venue_param['cross_street']
if 'state' in j_venue_param:
new_uservenue.state = j_venue_param['state']
if 'zip' in j_venue_param:
new_uservenue.zipcode = j_venue_param['zip']
if 'phone' in j_venue_param:
new_uservenue.phone = j_venue_param['phone']
new_uservenue.has_parent = True
new_uservenue.is_unique = is_unique_param
new_uservenue.checkin_list = checkin_list_param
new_uservenue.checkin_guid_list = checkin_guid_list_param
if not new_uservenue.checkin_guid_list or len(new_uservenue.checkin_guid_list) is 0:
new_uservenue.checkin_guid_list = [str(checkin_id) for checkin_id in new_uservenue.checkin_list]
return new_uservenue
new_uservenue.cross_street = j_venue_param['cross_street']
if 'state' in j_venue_param:
new_uservenue.state = j_venue_param['state']
if 'zip' in j_venue_param:
new_uservenue.zipcode = j_venue_param['zip']
if 'phone' in j_venue_param:
new_uservenue.phone = j_venue_param['phone']
new_uservenue.has_parent = True
new_uservenue.is_unique = is_unique_param
new_uservenue.checkin_list = checkin_list_param
new_uservenue.checkin_guid_list = checkin_guid_list_param
if not new_uservenue.checkin_guid_list or len(new_uservenue.checkin_guid_list) is 0:
new_uservenue.checkin_guid_list = [str(checkin_id) for checkin_id in new_uservenue.checkin_list]
return new_uservenue
uservenue = UserVenue.all().filter('user = '******'venue_guid = ', str(j_venue['id'])).filter('has_parent = ', True).get()
if uservenue:
if not uservenue.checkin_guid_list or len(uservenue.checkin_guid_list) is 0:
uservenue.checkin_guid_list = [str(checkin_id) for checkin_id in uservenue.checkin_list]
else:
uservenue = UserVenue.all().filter('user ='******'venue_guid =', str(j_venue['id'])).get()
if uservenue:
uservenue = uservenue_factory(userinfo, j_venue, uservenue.checkin_guid_list, uservenue.checkin_list, False)
else:
uservenue = UserVenue.all().filter('user ='******'venue_id =', j_venue['id']).get()
if uservenue:
uservenue = uservenue_factory(userinfo, j_venue, uservenue.checkin_guid_list, uservenue.checkin_list, False)
else:
userinfo.venue_count = userinfo.venue_count + 1
uservenue = uservenue_factory(userinfo, j_venue, [], [], True)
uservenue.last_updated = datetime.strptime(checkin['created'], "%a, %d %b %y %H:%M:%S +0000") #WARNING last_updated is confusing and should be last_checkin_at
# db.delete(mapimages)
# while(UserVenue.all().count() > 0):
# uservenues = UserVenue.all().fetch(500)
# db.delete(uservenues)
# while(AuthToken.all().count() > 0):
# authtokens = AuthToken.all().fetch(500)
# db.delete(authtokens)
# while(UserInfo.all().count() > 0):
# userinfos = UserInfo.all().fetch(500)
# db.delete(userinfos)
elif rest == 'user':
user = users.get_current_user()
if user:
while(MapImage.all().filter('user ='******'user ='******'user = '******'user = '******'user ='******'user ='******'mapimage':
user = users.get_current_user()
if user:
while(MapImage.all().filter('user ='******'user =', user).fetch(500)
db.delete(mapimages)
if not 'checkins' in history:
if 'unauthorized' in history:
userinfo.is_authorized = False
return 0
else:
logging.warning("no value for 'checkins' or 'unauthorized' in history: " + str(history))
return -1
elif history['checkins'] == None:
return 0
userinfo.put()
for checkin in history['checkins']:
if 'venue' in checkin:
j_venue = checkin['venue']
if 'id' in j_venue and 'geolat' in j_venue and 'geolong' in j_venue:
uservenue = UserVenue.all().filter('user ='******'venue_id =', j_venue['id']).get()
if uservenue == None:
uservenue = UserVenue(location = db.GeoPt(j_venue['geolat'], j_venue['geolong']))
uservenue.update_location()
uservenue.user = userinfo.user
userinfo.venue_count = userinfo.venue_count + 1
uservenue.venue_id = int(j_venue['id'])
if 'name' in j_venue:
uservenue.name = j_venue['name']
try:
if 'address' in j_venue:
uservenue.address = j_venue['address']
except BadValueError:
logging.error("Address not added for venue %s with address json '%s'" % (j_venue['id'], j_venue['address']))
if 'cross_street' in j_venue:
uservenue.cross_street = j_venue['cross_street']
return 0, 0
try:
if not 'checkins' in history:
logging.error("no value for 'checkins' in history: " + str(history))
userinfo.put()
return -1, 0
elif history['checkins']['items'] == None:
userinfo.put()
return 0, 0
history['checkins']['items'].reverse()
logging.debug('will process %d items' % (len(history['checkins']['items'])))
for checkin in history['checkins']['items']:
if 'venue' in checkin:
j_venue = checkin['venue']
if 'id' in j_venue:
uservenue = UserVenue.all().filter('user = '******'venue_guid = ', str(j_venue['id'])).get()
if not uservenue and 'location' in j_venue and 'lat' in j_venue['location'] and 'lng' in j_venue['location']:
userinfo.venue_count = userinfo.venue_count + 1
uservenue = UserVenue(parent=userinfo, location = db.GeoPt(j_venue['location']['lat'], j_venue['location']['lng']))
uservenue.venue_guid = str(j_venue['id'])
uservenue.update_location()
uservenue.user = userinfo.user
uservenue.checkin_guid_list = []
if uservenue: # if there's no uservenue by this point, then the venue was missing a location
uservenue.checkin_guid_list.append(str(checkin['id']))
userinfo.checkin_count += 1
def put_updated_uservenue_and_userinfo(uservenue_param, userinfo_param, num_added):
uservenue_param.put()
userinfo_param.put()
return num_added + 1
try:
import logging
from google.appengine.ext import db
from models import UserVenue
from google.appengine.runtime import DeadlineExceededError
total_deleted = 0
try:
child_uservenues = UserVenue.all().filter('has_parent = ', True).filter('is_unique = ', False).fetch(500)
for child_uservenue in child_uservenues:
orphan_uservenues = UserVenue.all().filter('venue_guid =', child_uservenue.venue_guid).fetch(500)
for orphan_uservenue in orphan_uservenues:
if not orphan_uservenue.has_parent or orphan_uservenue.has_parent == False:
orphan_uservenue.delete()
total_deleted += 1
child_uservenue.is_unique = True
child_uservenue.put()
logging.info("deleted a total of %d orphaned uservenues" % total_deleted)
except DeadlineExceededError:
logging.info("deleted a total of %d orphaned uservenues (DeadlineExceededError, but it's fine)" % total_deleted)
return 0, 0
try:
if not 'checkins' in history:
logging.error("no value for 'checkins' in history: " + str(history))
userinfo.put()
return -1, 0
elif history['checkins']['items'] == None:
userinfo.put()
return 0, 0
history['checkins']['items'].reverse()
logging.debug('will process %d items' % (len(history['checkins']['items'])))
for checkin in history['checkins']['items']:
if 'venue' in checkin:
j_venue = checkin['venue']
if 'id' in j_venue:
uservenue = UserVenue.all().filter('user = '******'venue_guid = ', str(j_venue['id'])).get()
if not uservenue and 'location' in j_venue and 'lat' in j_venue['location'] and 'lng' in j_venue['location']:
userinfo.venue_count = userinfo.venue_count + 1
uservenue = UserVenue(parent=userinfo, location = db.GeoPt(j_venue['location']['lat'], j_venue['location']['lng']))
uservenue.venue_guid = str(j_venue['id'])
uservenue.update_location()
uservenue.user = userinfo.user
uservenue.checkin_guid_list = []
if uservenue: # if there's no uservenue by this point, then the venue was missing a location
uservenue.checkin_guid_list.append(str(checkin['id']))
userinfo.checkin_count += 1
def put_updated_uservenue_and_userinfo(uservenue_param, userinfo_param, num_added):
uservenue_param.put()
userinfo_param.put()
return num_added + 1
try: