def post(self):
# Check parameters
name = self.request.get('name', None)
if not name:
self.error(400)
return
source_name = self.request.get('source', None)
# Check cache
key = self.cache_key(name, source_name)
names = cache.get(key)
if names:
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(names)
return
source = sources.get(source_name)
if not source:
# TODO: Get names from all sources?
self.error(404)
return
# Make service request for names
names, status_code = self.get_names(source, name)
if status_code != 200:
self.error(status_code)
return
# Update cache and send response
cache.add(key, names)
self.response.set_status(200)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write(simplejson.dumps(names))
def post(self):
# Parameters checked by frontend
name = self.request.get('name')
source_name = self.request.get('source')
# Get source
source = sources.get(source_name)
if not source:
logging.error('Cannot harvest without a source')
self.error(404)
# Update job status to 'error'
job = get_job(name, source_name, 'error', msg='Unsupported source')
cache.add(key, job)
return
# Check cache for harvest job
key = get_job_cache_key(name, source_name)
job = cache.get(key)
if not job:
self.error(404)
self.response.headers['Content-Type'] = "application/json"
self.response.out.write('{"error":"unknown job %s"}' % key)
return
count = 0
# Update job status to 'working'
cache.add(key, get_job(name, source_name, 'working', msg=count))
# Get points from source and put them into datastore in batches
pcount = 0
for points in self.get_points(name, source):
logging.info('HARVEST BACKEND MEMORY = %s after %s points' % (runtime.memory_usage().current(), count))
entities = []
for p in points:
pkey = Key('Point', '%s-%s-%s' % (source_name, name, pcount))
pcount += 1
entities.append(Point(key=pkey, lat=p[0], lng=p[1]))
entities.append(PointIndex.create(pkey, p[0], p[1], name, source_name))
model.put_multi(entities)
count += len(points)
cache.add(key, get_job(name, source_name, 'working', msg=count))
# Update job status to 'done'
# TODO: Done now or after backend rendering completes?
cache.add(key, get_job(name, source_name, 'done', msg=count))
