# maps 1-to-1 to a Google Fusion Tables schema
SCHEMA = {
'Fragile Oasis':HashableOrderedDict([
('title', 'STRING'),
('category', 'STRING'),
('url', 'STRING'),
('website', 'STRING'),
('cords', 'LOCATION'),
('city', 'LOCATION'), # augmented with OpenStreetMap
('cityinfo', 'LOCATION'), # augmented with Freebase
('country', 'LOCATION'), # augmented with OpenStreetMap
('countryinfo', 'LOCATION'), # augmented with Freebase
('description', 'STRING'),
('votes', 'NUMBER'),
('location', 'LOCATION'),
('comments', 'NUMBER'),
('featured', 'STRING'),
('followers', 'NUMBER'),
('followersinfo', 'NUMBER'), # augmented by scraping fragileoasis.org
('shares', 'NUMBER'),
('id', 'NUMBER'),
('photos', 'STRING'),
])
}
if __name__ == '__main__':
pusher = FTPusher()
pusher.create_table(SCHEMA)
return _get_entity_info(city, '/location/citytown')
# get Country entity on Freebase
def _get_country_info(country):
return _get_entity_info(country, '/location/country')
# enhance the Fragile Oasis dataset with external data
def _enhance_FO(data):
for row in data:
city, country = _extract_city_country(row['cords'])
row['city'] = city
row['cityinfo'] = _get_city_info(city)
row['country'] = country
row['countryinfo'] = _get_country_info(country)
row['followersinfo'] = _scrape_followers(row['url'])
return data
def import_fragile_oasis():
r = requests.get('http://www.fragileoasis.org/projects/feed/?format=json')
data = json.load(r, object_hook=util.fragile_oasis_parser)
return _enhance_FO(data)
if __name__ == '__main__':
pusher = FTPusher()
pusher.populate_table(import_fragile_oasis())
