Beispiel #1
0
def load_vic_data():

    num = vic.get_number_of_places()
    print "Importing %d Victorian Heritage Places" % num

    for place in progress.bar(vic.all_places(), width=80, expected_size=num):
        try:
            add_heritage_place(place)
        except RequestError, err:
            print json.dumps(place, indent=2)
            print traceback.format_exc()

            raise RequestError
Beispiel #2
0
def stream_vic_data(index_name=None):
    if not index_name:
        index_name = current_app.config['ES_INDEX']
    ES_DOCTYPE = current_app.config['ES_DOCTYPE']

    num = vic.get_number_of_places()
    print "Importing %d Victorian Heritage Places" % num

    for ok, result in progress.bar(
                        elasticsearch.helpers.streaming_bulk(es.get_es(),
                            indexable_objects_iter(vic.all_places(),
                                index_name, ES_DOCTYPE
                            ), chunk_size=100
                      ), width=80, expected_size=num):
        index = result.get('index', {})
        status = index.get('status')
        if not ok and status != 200:
            action, result = result.popitem()
            doc_id = '/%s/commits/%s' % (index_name, result['_id'])
            print('Failed to %s document %s: %r' % (action, doc_id, result))