def load_vic_data():
num = vic.get_number_of_places()
print "Importing %d Victorian Heritage Places" % num
for place in progress.bar(vic.all_places(), width=80, expected_size=num):
try:
add_heritage_place(place)
except RequestError, err:
print json.dumps(place, indent=2)
print traceback.format_exc()
raise RequestError
def stream_vic_data(index_name=None):
if not index_name:
index_name = current_app.config['ES_INDEX']
ES_DOCTYPE = current_app.config['ES_DOCTYPE']
num = vic.get_number_of_places()
print "Importing %d Victorian Heritage Places" % num
for ok, result in progress.bar(
elasticsearch.helpers.streaming_bulk(es.get_es(),
indexable_objects_iter(vic.all_places(),
index_name, ES_DOCTYPE
), chunk_size=100
), width=80, expected_size=num):
index = result.get('index', {})
status = index.get('status')
if not ok and status != 200:
action, result = result.popitem()
doc_id = '/%s/commits/%s' % (index_name, result['_id'])
print('Failed to %s document %s: %r' % (action, doc_id, result))