def do_work(para): venue_id = para[0] crawl_user_id = para[1] drop_table_venue_photo_instagram(crawl_user_id) add_table_venue_photo_instagram(crawl_user_id) photos_gen = download_instagram_photos(venue_id, time.time()-24*3600*30, time.time()) for p in photos_gen: (data, venue_id, instagram_id) = p save_photo_instagram(data, venue_id, instagram_id, crawl_user_id)
def instagram_test(): add_table_venue_photo_instagram() crawler = VenuePhotoCrawlerInstagram() foursquare_ids = get_all_foursquare_ids() #fetched_ids = get_all_photo_fetched_venue_id_instagram() for foursquare_id in foursquare_ids.keys(): """NOTICE THIS IS TO AVOID REPEATING FETCHING""" #if foursquare_id not in fetched_ids: print foursquare_id, foursquare_ids[foursquare_id] crawler.grab_photos(foursquare_id)