Beispiel #1
0
def apt_list(request, city):
    limit = 3000
    query = Apartment.all()
    query.filter("region =", city.upper())
    query.order('-updated')
    bad_keys = ('price_thousands', 'region', 'updated_hour', 'updated_day',
                'addr', 'location_accuracy', 'geohash')

    results = []
    while True:
        queryresults = query.fetch(1000)
        if not queryresults:
            break
        for result in queryresults:
            d = dict(result._entity)
            for key in bad_keys:
                del d[key]
            for key, value in d.items():
                if isinstance(value, bool):
                    d[key] = int(value)
            d['updated'] = int(d['updated'].strftime("%s"))
            if d.get('location'):
                d['location'] = (d['location'].lat, d['location'].lon)
            else:
                continue
            results.append(d)
            if len(results) >= limit:
                break
        if len(results) >= limit:
            break

    return HttpResponse(simplejson.dumps({'results': results}),
                        mimetype='application/json')
Beispiel #2
0
def count_breakdowns(request):
    INTERVAL = 200
    try:
        count_data = pickle.loads(request.POST['count_data'].decode('base64'))
    except:
        count_data = defaultdict(int)

    idx = int(request.POST.get('idx', 0))
    if idx == -1:
        ApartmentCounts.update_data(count_data)
        return HttpResponse("Boo", mimetype="text/plain")

    cursor = request.POST.get('cursor')

    query = Apartment.all().order("updated")
    if cursor:
        query.with_cursor(cursor)

    results = query.fetch(INTERVAL)

    for apt in results:
        for field, mapper in field_to_str.items():
            count_data['%s__%s' % (field, mapper(getattr(apt, field)))] += 1

    createstr = "Not creating"

    output = "%s\n\n%s" % (createstr, pprint.pformat(dict(count_data)))

    if len(results) == INTERVAL:
        idx += INTERVAL
        cursor = query.cursor()
        cdata = pickle.dumps(count_data).encode('base64')
        params = {'cursor': cursor, 'idx': idx, 'count_data': cdata}

        taskqueue.add(url="/apts/breakdown/", params=params,
                      name="Counter-%s-%s" % (idx, randstr()),
                      method="POST")

        createstr = "Spawning new process..."
    else:
        idx = -1
        cdata = pickle.dumps(count_data).encode('base64')
        params = {'idx': -1, 'count_data': cdata}
        taskqueue.add(url="/apts/breakdown/", params=params,
                      name="Counter-%s-%s" % (idx, randstr()),
                      method="POST")

    return HttpResponse(output, mimetype="text/plain")
Beispiel #3
0
def clean_apts(request):
    NUM = int(request.GET.get('NUM', 300))
    COUNT = int(request.GET.get('COUNT', 0))
    SPAWN = int(request.GET.get('SPAWN', 0))
    nospawn = int(request.GET.get('CHILD', 0))

    if SPAWN:
        for i in range(int(math.ceil(NUM / 100.0))):
            COUNT = i * 100
            taskqueue.add(url="/apts/clean/", params={"NUM": NUM, "COUNT": COUNT, 'CHILD': '1'},
                          name="DELETE-%s-%s-spawn-%s" % (NUM, COUNT, randstr()),
                          method="GET")
        return HttpResponse("Spawned a lot of processes...", mimetype="text/plain")

    start = time.time()
    try:
        num_deleted = Apartment.delete_some(100, 30)
    except Exception, e:
        result = "Error: %s" % e