def post(self):
num = 0
for stop in settings.STOP_TYPES:
counter = StopMeta.get(Key.from_path('StopMeta', 1))
q = db.Query(Stop, keys_only=True).filter("stop_type =", stop).fetch(100)
logging.debug(q)
counter.counter_delta(-len(q), stop, "NO")
num += len(q)
db.delete(q)
q = db.Query(StopUpdate, keys_only=True).filter("stop_type =", stop).fetch(100)
logging.debug(q)
counter.counter_delta(-len(q), stop, "UPDATE")
num += len(q)
db.delete(q)
q = db.Query(StopNew, keys_only=True).filter("stop_type =", stop).fetch(100)
logging.debug(q)
counter.counter_delta(-len(q), stop, "NEW")
num += len(q)
db.delete(q)
counter.put()
if num:
taskqueue.add(url='/purge/delete', queue_name='import')
memcache.set('import_status', "Deletion finished.", time=30)
def get(self, login_user=None, template_values={}):
# is import or update going on?
template_values['update_status'] = memcache.get('update_status')
template_values['import_status'] = memcache.get('import_status')
# Check for existing data.
try:
counter = StopMeta.get(Key.from_path('StopMeta', 1))
template_values['production_num_stops'] = counter.counter_stop_no_confirm
template_values['production_num_stations'] = counter.counter_station_no_confirm
template_values['production_num_places'] = counter.counter_place_no_confirm
# confirmation outstanding
template_values['update_num_stops'] = counter.counter_stop_update_confirm
template_values['update_num_stations'] = counter.counter_station_update_confirm
template_values['update_num_places'] = counter.counter_place_update_confirm
template_values['new_num_stops'] = counter.counter_stop_new_confirm
template_values['new_num_stations'] = counter.counter_station_new_confirm
template_values['new_num_places'] = counter.counter_place_new_confirm
# Administrative hierarchy
template_values['gov_num'] = Comuna.all().count()+Region.all().count()+Country.all().count()
except AttributeError:
# no data in database. Redirect to import page
self.redirect('/import')
template_values['upload_url'] = blobstore.create_upload_url('/update/upload')
path = os.path.join(os.path.dirname(__file__), "pages/update.html")
self.response.out.write(template.render(path, template_values))
return
def get(self, login_user=None, template_values={}):
# hold update query in memcache
q_new = memcache.get('new')
if not q_new:
q_new = Stop.all().filter('confirm =', 'NEW')
memcache.set('new', q_new)
offset = int(self.request.get("offset", "0"))
new = q_new.fetch(limit=1, offset=offset)
template_values['offset'] = offset+1
if not new:
# nothing to do
self.redirect('/update')
return
new = new[0]
# find datasets nearby
query = Stop.all().filter('confirm =', 'NO')
proximity = geo.geomodel.GeoModel.proximity_fetch(query, new.location, max_results=6, max_distance=500)
compare=[]
stop = {}
stop['description'] = 'New:'
stop['type'] = new.stop_type
stop['name'] = ", ".join(new.names)
stop['lat'] = new.location.lat
stop['lon'] = new.location.lon
stop['new'] = True
stop['key'] = str(new.key())
compare.append(stop)
for pstop in proximity:
stop = {}
# calculate distance to new point
dist = geo.geomath.distance(new.location,pstop.location)
# calculate 'distance' between names (ratio of similarity)
ndiff = difflib.SequenceMatcher(None,"".join(new.names),"".join(pstop.names))
logging.debug("%s vs. %s ratio %f" % ("".join(new.names),"".join(pstop.names),ndiff.ratio()))
if dist < 100.0 or ndiff.ratio() > 0.7:
stop['checked'] = "checked"
stop['style'] = "balsa-watch"
stop['description'] = 'at %4.0f meters' % (dist)
stop['type'] = pstop.stop_type
stop['name'] = ", ".join(pstop.names)
stop['lat'] = pstop.location.lat
stop['lon'] = pstop.location.lon
stop['new'] = False
stop['key'] = str(pstop.key())
compare.append(stop)
template_values['compare'] = compare
counter = StopMeta.all().get()
template_values['new_num_stops'] = counter.counter_stop_new_confirm
template_values['new_num_stations'] = counter.counter_station_new_confirm
template_values['new_num_places'] = counter.counter_place_new_confirm
path = os.path.join(os.path.dirname(__file__), "pages/confirm_new.html")
self.response.out.write(template.render(path, template_values))
return
def update_counter(cls):
"""Update meta counter fields in transaction"""
counter = StopMeta.get(Key.from_path('StopMeta', 1))
for stop in cls._stop_data:
if isinstance(stop, Stop):
counter.counter_delta(1, stop.stop_type, "NO")
elif isinstance(stop, StopUpdate):
counter.counter_delta(1, stop.stop_type, "UPDATE")
elif isinstance(stop, StopNew):
counter.counter_delta(1, stop.stop_type, "NEW")
else:
assert False, "Unknown stop instance: %s" % (stop)
if cls._timestamp > counter.last_update:
counter.last_update = cls._timestamp
counter.put()
def get(self, login_user=None, template_values={}):
# Check for existing data. Import is only done with an empty database
# Note: If an area is imported which has no intersection with the existing
# data it is a differrent matter. This will be handled in proper
# time. For the moment we concentrate on Chile.
if StopMeta.get(Key.from_path('StopMeta', 1)):
self.redirect('/update')
# set counter fields to zero
# counter_fields = StopMeta(Key.from_path('StopMeta', 1))
def store():
counter_fields = StopMeta(key=Key.from_path('StopMeta', 1))
counter_fields.zero_all()
counter_fields.last_update = datetime.datetime(2011,1,1)
counter_fields.put()
db.run_in_transaction(store)
template_values['upload_url'] = blobstore.create_upload_url('/import/upload')
path = os.path.join(os.path.dirname(__file__), "pages/import_select.html")
self.response.out.write(template.render(path, template_values))
return
def get(self, login_user=None, template_values={}):
obsolete = [key for key in self.request.get_all("obsolete_key", None)]
logging.debug(obsolete)
key = self.request.get("accept", None)
assert key, "Did not receive key"
new_stop = Stop.get(key)
new_stop.confirm = "NO"
counter = StopMeta.all().get()
def store():
for key in obsolete:
stop = Stop.all().ancestor(counter).filter("__key__ =", Key(key)).get()
counter.counter_delta(-1, stop.stop_type)
counter.counter_delta(1, stop.stop_type)
counter.counter_delta(-1, stop.stop_type, "NEW")
db.put(new_stop)
db.delete(obsolete)
counter.put()
db.run_in_transaction(store)
self.redirect('/update/confirm/new')
def get(self, login_user=None, template_values={}):
pass
# hold update query in memcache
q_update = memcache.get('update')
if not q_update:
q_update = Stop.all().filter('confirm =', 'UPDATE')
memcache.set('update', q_update)
offset = int(self.request.get("offset", "0"))
update = q_update.fetch(limit=1, offset=offset)
template_values['offset'] = offset+1
if not update:
# nothing to do
self.redirect('/update')
return
update = update[0]
# get corresponding production data
production = Stop.all().filter('osm_id =', update.osm_id).filter('confirm =', 'NO').get()
assert production, "Failed to fetch prodcution data for update osm_id=%d" % (update.osm_id)
compare={}
compare['key'] = str(update.key())
compare['data'] = []
data = {}
data['type'] = 'Stop type'
data['production'] = production.stop_type
data['update'] = update.stop_type
if production.stop_type != update.stop_type:
data['style'] = "balsa-watch"
compare['data'].append(data)
data = {}
data['type'] = 'Location (lat)'
data['production'] = production.location.lat
data['update'] = update.location.lat
if production.location.lat != update.location.lat:
data['style'] = "balsa-watch"
compare['data'].append(data)
data = {}
data['type'] = 'Location (lon)'
data['production'] = production.location.lon
data['update'] = update.location.lon
if production.location.lon != update.location.lon:
data['style'] = "balsa-watch"
compare['data'].append(data)
data = {}
data['type'] = 'Name(s)'
data['production'] = ", ".join(production.names)
data['update'] = ", ".join(update.names)
if production.names != update.names:
data['style'] = "balsa-watch"
compare['data'].append(data)
data = {}
data['type'] = 'Location zoom'
data['production'] = ", ".join(production.gov.gov_names)
data['update'] = ", ".join(update.gov.gov_names)
if production.gov.gov_names != update.gov.gov_names:
data['style'] = "balsa-watch"
compare['data'].append(data)
template_values['compare'] = compare
counter = StopMeta.all().get()
template_values['update_num_stops'] = counter.counter_stop_update_confirm
template_values['update_num_stations'] = counter.counter_station_update_confirm
template_values['update_num_places'] = counter.counter_place_update_confirm
path = os.path.join(os.path.dirname(__file__), "pages/confirm_update.html")
self.response.out.write(template.render(path, template_values))
return
def store():
counter_fields = StopMeta(key=Key.from_path('StopMeta', 1))
counter_fields.zero_all()
counter_fields.last_update = datetime.datetime(2011,1,1)
counter_fields.put()