def update_airports(): """Get the city from Latte and update airport.""" # build bulk from ghost_spider.elastic import LocationAirportEs from ghost_spider import progressbar import requests import time size = 100 page = 0 progress = None total = 0 query = {"query": {"match_all": {}}, "size": size, "from": 0} query["sort"] = [{"name.untouched": "asc"}] while True: query["from"] = page * size results = LocationAirportEs.search(query) if not progress: total = results["hits"]["total"] print "total %s" % total progress = progressbar.AnimatedProgressBar(end=total, width=100) progress.show_progress() if not results["hits"]["hits"]: break page += 1 bulk = "" for result in results["hits"]["hits"]: location = result["_source"] data_id = result["_id"] if not location.get('place_id'): ret_json = [] url = u'https://latte.la/travel/json?term=%s&kind=area1-area2-area3-city1-city2-city3' % location["area"] ret = requests.get(url, params={}) ret_json = ret.json() found_place = None progress + 1 progress.show_progress() for place in ret_json: if place.get('value') == location["area"]: found_place = place break if found_place: location["place_id"] = found_place["placeid"] location["place_url_key"] = found_place["url_key"] else: location["place_url_key"] = location["url_key"] location["place_id"] = long(location.get('place_id') or 0) location.pop('url_key') bulk += LocationAirportEs.bulk_data(location, action="update", data_id=data_id) if bulk: LocationAirportEs.send(bulk) if progress: progress + total progress.show_progress() print " "