def test_writer(): path = jsonl_file_path(datetime.datetime(2001, 3, 27), pref='test_') try: os.remove(path) except OSError: pass wtr = Writer(path) scr = mocked_scraper() wtr.write(scr.extract_data()[3:7], file_close=True) assert 4 == sum(1 for _ in open(path, 'r')), 'written 4 lines'
def handle(self, *args, **options): biz_date = to_biz_date(datetime.datetime.now()) file_path = jsonl_file_path(biz_date) writer = Writer(file_path) for shop in Shop.scrapeable(): print shop.id # TODO django-logging page = Page(shop) scr = Scraper(page) data = scr.extract_data() writer.write(data) writer.file.close()