def apt_list(request, city):
limit = 3000
query = Apartment.all()
query.filter("region =", city.upper())
query.order('-updated')
bad_keys = ('price_thousands', 'region', 'updated_hour', 'updated_day',
'addr', 'location_accuracy', 'geohash')
results = []
while True:
queryresults = query.fetch(1000)
if not queryresults:
break
for result in queryresults:
d = dict(result._entity)
for key in bad_keys:
del d[key]
for key, value in d.items():
if isinstance(value, bool):
d[key] = int(value)
d['updated'] = int(d['updated'].strftime("%s"))
if d.get('location'):
d['location'] = (d['location'].lat, d['location'].lon)
else:
continue
results.append(d)
if len(results) >= limit:
break
if len(results) >= limit:
break
return HttpResponse(simplejson.dumps({'results': results}),
mimetype='application/json')
def count_breakdowns(request):
INTERVAL = 200
try:
count_data = pickle.loads(request.POST['count_data'].decode('base64'))
except:
count_data = defaultdict(int)
idx = int(request.POST.get('idx', 0))
if idx == -1:
ApartmentCounts.update_data(count_data)
return HttpResponse("Boo", mimetype="text/plain")
cursor = request.POST.get('cursor')
query = Apartment.all().order("updated")
if cursor:
query.with_cursor(cursor)
results = query.fetch(INTERVAL)
for apt in results:
for field, mapper in field_to_str.items():
count_data['%s__%s' % (field, mapper(getattr(apt, field)))] += 1
createstr = "Not creating"
output = "%s\n\n%s" % (createstr, pprint.pformat(dict(count_data)))
if len(results) == INTERVAL:
idx += INTERVAL
cursor = query.cursor()
cdata = pickle.dumps(count_data).encode('base64')
params = {'cursor': cursor, 'idx': idx, 'count_data': cdata}
taskqueue.add(url="/apts/breakdown/", params=params,
name="Counter-%s-%s" % (idx, randstr()),
method="POST")
createstr = "Spawning new process..."
else:
idx = -1
cdata = pickle.dumps(count_data).encode('base64')
params = {'idx': -1, 'count_data': cdata}
taskqueue.add(url="/apts/breakdown/", params=params,
name="Counter-%s-%s" % (idx, randstr()),
method="POST")
return HttpResponse(output, mimetype="text/plain")
def clean_apts(request):
NUM = int(request.GET.get('NUM', 300))
COUNT = int(request.GET.get('COUNT', 0))
SPAWN = int(request.GET.get('SPAWN', 0))
nospawn = int(request.GET.get('CHILD', 0))
if SPAWN:
for i in range(int(math.ceil(NUM / 100.0))):
COUNT = i * 100
taskqueue.add(url="/apts/clean/", params={"NUM": NUM, "COUNT": COUNT, 'CHILD': '1'},
name="DELETE-%s-%s-spawn-%s" % (NUM, COUNT, randstr()),
method="GET")
return HttpResponse("Spawned a lot of processes...", mimetype="text/plain")
start = time.time()
try:
num_deleted = Apartment.delete_some(100, 30)
except Exception, e:
result = "Error: %s" % e