def test_writer():
path = jsonl_file_path(datetime.datetime(2001, 3, 27), pref='test_')
try:
os.remove(path)
except OSError:
pass
wtr = Writer(path)
scr = mocked_scraper()
wtr.write(scr.extract_data()[3:7], file_close=True)
assert 4 == sum(1 for _ in open(path, 'r')), 'written 4 lines'
def handle(self, *args, **options):
biz_date = to_biz_date(datetime.datetime.now())
file_path = jsonl_file_path(biz_date)
reader = Reader(file_path)
organizer = Organizer()
for data in reader.readlines():
try:
organizer.organize(data)
except InvalidDataException:
print "invalid-data\t" + str(data)
def handle(self, *args, **options):
biz_date = to_biz_date(datetime.datetime.now())
file_path = jsonl_file_path(biz_date)
writer = Writer(file_path)
for shop in Shop.scrapeable():
print shop.id # TODO django-logging
page = Page(shop)
scr = Scraper(page)
data = scr.extract_data()
writer.write(data)
writer.file.close()
def test_jsonl_file_path():
assert jsonl_file_path(datetime.date(2016, 3, 27), 'z') == "./resources/jsonl/z20160327.jsonl"
