def spider_opened(self, spider):
'''Initialize your exporters when spider first starts
'''
log.msg(
'\n\n{0}\n{1}\n{2}\n\n'.format('-' * 79, spider.name, '-' * 79),
log.INFO)
# figure out which fields to export based on the spiders name
# this also exports the fields in the specified order
if spider.name == 'YelpSpider':
foo_fields = ['biz_id', 'timestamp', 'stars', 'review_count']
elif spider.name == 'AptratingsSpider':
foo_fields = [
'biz_id', 'recommended_by', 'total_overall_rating',
'overall_parking', 'overall_maintenance',
'overall_construction', 'overall_noise', 'overall_grounds',
'overall_safety', 'overall_office_staff', 'comment_url',
'comment_title', 'comment_username', 'comment_post_date',
'comment_years_stayed', 'comment_message', 'last_edited',
'replies', 'overall_rating', 'parking', 'maintenance',
'construction', 'noise', 'grounds', 'safety', 'office_staff'
]
else:
raise FeverException('a foo_fields variable is not yet \
defined for "{0}"'.format(spider.name))
# create a unique filename
self.file = open(
os.path.join('data', make_filename(spider.name, 'json')), 'w+b')
self.json_exporter = FeverJsonItemExporter(self.file,
fields_to_export=foo_fields,
sort_keys=True,
indent=2)
# start exporting items to JSON file
self.json_exporter.start_exporting()
