def apt_list(request, city): limit = 3000 query = Apartment.all() query.filter("region =", city.upper()) query.order('-updated') bad_keys = ('price_thousands', 'region', 'updated_hour', 'updated_day', 'addr', 'location_accuracy', 'geohash') results = [] while True: queryresults = query.fetch(1000) if not queryresults: break for result in queryresults: d = dict(result._entity) for key in bad_keys: del d[key] for key, value in d.items(): if isinstance(value, bool): d[key] = int(value) d['updated'] = int(d['updated'].strftime("%s")) if d.get('location'): d['location'] = (d['location'].lat, d['location'].lon) else: continue results.append(d) if len(results) >= limit: break if len(results) >= limit: break return HttpResponse(simplejson.dumps({'results': results}), mimetype='application/json')
def count_breakdowns(request): INTERVAL = 200 try: count_data = pickle.loads(request.POST['count_data'].decode('base64')) except: count_data = defaultdict(int) idx = int(request.POST.get('idx', 0)) if idx == -1: ApartmentCounts.update_data(count_data) return HttpResponse("Boo", mimetype="text/plain") cursor = request.POST.get('cursor') query = Apartment.all().order("updated") if cursor: query.with_cursor(cursor) results = query.fetch(INTERVAL) for apt in results: for field, mapper in field_to_str.items(): count_data['%s__%s' % (field, mapper(getattr(apt, field)))] += 1 createstr = "Not creating" output = "%s\n\n%s" % (createstr, pprint.pformat(dict(count_data))) if len(results) == INTERVAL: idx += INTERVAL cursor = query.cursor() cdata = pickle.dumps(count_data).encode('base64') params = {'cursor': cursor, 'idx': idx, 'count_data': cdata} taskqueue.add(url="/apts/breakdown/", params=params, name="Counter-%s-%s" % (idx, randstr()), method="POST") createstr = "Spawning new process..." else: idx = -1 cdata = pickle.dumps(count_data).encode('base64') params = {'idx': -1, 'count_data': cdata} taskqueue.add(url="/apts/breakdown/", params=params, name="Counter-%s-%s" % (idx, randstr()), method="POST") return HttpResponse(output, mimetype="text/plain")