def _dump(self, type='ids'):
if type == 'ids':
log.info('dumping ids to %s' % config.flixster.titles_path)
json.dump(list(self.known_ids),
open(config.flixster.title_ids_path, 'w'))
elif type == 'reviews':
log.info('dumping reviews to %s' %
config.flixster.title_reviews_path)
dson.dump(dict([ (i['id'], i) for i in
self._get_fetched_info('reviews') ]),
config.flixster.titles_path)
else:
log.info('dumping titles to %s' % config.flixster.titles_path)
dson.dump(dict([ (i['id'], i) for i in
self._get_fetched_info('title') ]),
config.flixster.titles_path)
log.info('dump complete')
def _scrape_response(cls):
log.info('Done scraping! Dumping now...')
votes = {}
for data in filmdata.sink.get_source_data('netflix', 'title'):
votes[data['id']] = data['votes']
dson.dump(votes, config.netflix.votes_path)
def _scrape_response(cls, type='title'):
log.info('Done scraping! Dumping now...')
dson.dump(cls._get_known_ids(type), config.imdb['%s_ids_path' % type])