# maps 1-to-1 to a Google Fusion Tables schema SCHEMA = { 'Fragile Oasis':HashableOrderedDict([ ('title', 'STRING'), ('category', 'STRING'), ('url', 'STRING'), ('website', 'STRING'), ('cords', 'LOCATION'), ('city', 'LOCATION'), # augmented with OpenStreetMap ('cityinfo', 'LOCATION'), # augmented with Freebase ('country', 'LOCATION'), # augmented with OpenStreetMap ('countryinfo', 'LOCATION'), # augmented with Freebase ('description', 'STRING'), ('votes', 'NUMBER'), ('location', 'LOCATION'), ('comments', 'NUMBER'), ('featured', 'STRING'), ('followers', 'NUMBER'), ('followersinfo', 'NUMBER'), # augmented by scraping fragileoasis.org ('shares', 'NUMBER'), ('id', 'NUMBER'), ('photos', 'STRING'), ]) } if __name__ == '__main__': pusher = FTPusher() pusher.create_table(SCHEMA)
return _get_entity_info(city, '/location/citytown') # get Country entity on Freebase def _get_country_info(country): return _get_entity_info(country, '/location/country') # enhance the Fragile Oasis dataset with external data def _enhance_FO(data): for row in data: city, country = _extract_city_country(row['cords']) row['city'] = city row['cityinfo'] = _get_city_info(city) row['country'] = country row['countryinfo'] = _get_country_info(country) row['followersinfo'] = _scrape_followers(row['url']) return data def import_fragile_oasis(): r = requests.get('http://www.fragileoasis.org/projects/feed/?format=json') data = json.load(r, object_hook=util.fragile_oasis_parser) return _enhance_FO(data) if __name__ == '__main__': pusher = FTPusher() pusher.populate_table(import_fragile_oasis())