return
if not offer.get('location'):
location = geocode_city_country(offer['city'].lower(), offer['country'].lower()) # Lower for better cache
offer['location'] = location or None
Offer.upsert(session, offer)
@memory.cache()
def geocode_city_country(city, country):
location = '{city}, {country}'.format(city=city, country=country)
return geocoder.google(location, key=settings.API_KEY_GOOGLE_GEOCODER).wkt
# 1) Fetch the number of pages of offers
print("Fetching total info...")
max_page = Scraper.scrap_offers_list_max_page()
print("Total %d pages to fetch" % max_page)
# 2) Fetch the list of offer ids
thread_pool = ThreadPool(processes=cpu_count())
offer_ids = thread_pool.map(Scraper.scrap_offers_list_page, range(1, max_page+1))
thread_pool.close()
thread_pool.join()
offer_ids = list(sum(offer_ids, []))
print("OFFER IDS", len(offer_ids), offer_ids)
# 3) Fetcb all the offers
thread_pool = ThreadPool(processes=cpu_count())
offers = thread_pool.map(fetch_and_process_offer, offer_ids)
thread_pool.close()
