def stream_vic_data(index_name=None):
if not index_name:
index_name = current_app.config['ES_INDEX']
ES_DOCTYPE = current_app.config['ES_DOCTYPE']
num = vic.get_number_of_places()
print "Importing %d Victorian Heritage Places" % num
for ok, result in progress.bar(
elasticsearch.helpers.streaming_bulk(es.get_es(),
indexable_objects_iter(vic.all_places(),
index_name, ES_DOCTYPE
), chunk_size=100
), width=80, expected_size=num):
index = result.get('index', {})
status = index.get('status')
if not ok and status != 200:
action, result = result.popitem()
doc_id = '/%s/commits/%s' % (index_name, result['_id'])
print('Failed to %s document %s: %r' % (action, doc_id, result))
def add_heritage_place(place):
"""Add a heritage place to the search index
:param place: Dictionary defining a heritage place.
"""
ES_INDEX = current_app.config['ES_INDEX']
ES_DOCTYPE = current_app.config['ES_DOCTYPE']
try:
id = "%s-%s" % (place['state'], place['id'])
connection = es.get_es()
result = connection.index(index=ES_INDEX, doc_type=ES_DOCTYPE,
id=id, body=place)
# print result
except AttributeError as e:
print e
print place
return False
return True
def create_index(index_name):
mapping_body = {
"mappings": {
"heritage_place": {
"_source": {
"enabled": True
},
"_all": {
"enabled": True
},
"properties": {
"architectural_styles": {
"type": "string",
"fields" : {
"raw" : {"type" : "string", "index" : "not_analyzed"}
}
},
"architects": {
"type": "string",
"fields" : {
"raw" : {"type" : "string", "index" : "not_analyzed"}
}
},
"categories": {
"type": "object",
"properties": {
"name": {"type" : "string", "index" : "not_analyzed"},
"group": {"type" : "string", "index" : "not_analyzed"}
}
},
"date_created": {
"type": "string",
"index": "not_analyzed"
},
"date_modified": {
"type": "string",
"index": "not_analyzed"
},
"lga": {
"type": "string",
"index": "not_analyzed"
},
"place_details_url": {
"type": "string",
"index": "not_analyzed"
},
"url": {
"type": "string",
"index": "not_analyzed"
},
"vhr_number": {
"type": "string"
},
"status_name": {
"type": "string",
"index": "not_analyzed"
},
"geolocation": {
"type": "geo_point",
"lat_lon": True,
"geohash": True
},
"address": { "type": "string" },
"addresses": {
"type": "object",
"properties": {
"street_number": { "type": "string", "copy_to": "address" },
"address": { "type": "string", "copy_to": "address" },
"street_name": { "type": "string", "copy_to": "address" },
"suburb": { "type": "string", "index": "not_analyzed", "copy_to": "address" },
"state": { "type": "string", "copy_to": "address" },
"country": { "type": "string", "copy_to": "address" },
"postcode": { "type": "string", "copy_to": "address" },
"lga_name": { "type": "string", "index": "not_analyzed", "copy_to": "address" },
}
}
}
}
}
}
connection = es.get_es()
return connection.indices.create(index_name, mapping_body)
def reindex(source, target):
"""ReIndex data from source into target
Used when a new mapping has been created
"""
return elasticsearch.helpers.reindex(es.get_es(), source, target)
def get_index_version():
index_name = current_app.config['ES_INDEX']
connection = es.get_es()
return connection.indices.get_alias(index_name)
def delete_index(index_name):
"""Delete the entire index of heritage places - DANGER!!
"""
connection = es.get_es()
return connection.indices.delete(index_name)
def update_alias(index_name):
connection = es.get_es()
alias_name = current_app.config['ES_ALIAS']
print connection.indices.delete_alias(name=alias_name, index="_all")
return connection.indices.put_alias(index=index_name, name=alias_name)
