Пример #1
0
 def post(self):
     # Check parameters
     name = self.request.get('name', None)
     if not name:
         self.error(400)
         return
     source_name = self.request.get('source', None)
     
     # Check cache
     key = self.cache_key(name, source_name)
     names = cache.get(key)
     if names:
         self.response.set_status(200)
         self.response.headers['Content-Type'] = "application/json"
         self.response.out.write(names)
         return
     
     source = sources.get(source_name)
     if not source:
         # TODO: Get names from all sources?
         self.error(404)
         return
     
     # Make service request for names
     names, status_code = self.get_names(source, name)        
     if status_code != 200:
         self.error(status_code)
         return
     
     # Update cache and send response
     cache.add(key, names)
     self.response.set_status(200)
     self.response.headers['Content-Type'] = "application/json"
     self.response.out.write(simplejson.dumps(names))
Пример #2
0
    def post(self):
        # Parameters checked by frontend
        name = self.request.get('name')
        source_name = self.request.get('source')

        # Get source
        source = sources.get(source_name)
        if not source:
            logging.error('Cannot harvest without a source')
            self.error(404)
            # Update job status to 'error'
            job = get_job(name, source_name, 'error', msg='Unsupported source')
            cache.add(key, job)
            return

        # Check cache for harvest job 
        key = get_job_cache_key(name, source_name)
        job = cache.get(key)
        if not job:
                self.error(404)
                self.response.headers['Content-Type'] = "application/json"
                self.response.out.write('{"error":"unknown job %s"}' % key)
                return
            
        count = 0

        # Update job status to 'working'
        cache.add(key, get_job(name, source_name, 'working', msg=count))       
        
        # Get points from source and put them into datastore in batches
        pcount = 0
        for points in self.get_points(name, source):
            logging.info('HARVEST BACKEND MEMORY = %s after %s points' % (runtime.memory_usage().current(), count))
            entities = []
            for p in points:
                pkey = Key('Point', '%s-%s-%s' % (source_name, name, pcount))
                pcount += 1
                entities.append(Point(key=pkey, lat=p[0], lng=p[1]))
                entities.append(PointIndex.create(pkey, p[0], p[1], name, source_name))
            model.put_multi(entities)
            count += len(points)
            cache.add(key, get_job(name, source_name, 'working', msg=count))

        # Update job status to 'done'
        # TODO: Done now or after backend rendering completes?
        cache.add(key, get_job(name, source_name, 'done', msg=count))