def main(fd): engine = get_db_engine_for_pandas() total_page = int(get_page_num(get_suspended_url(fd=fd))['pages']) loop = asyncio.get_event_loop() tasks = [ loop.create_task(parse(get_suspended_url(fd=fd, page=p), engine)) for p in range(1, total_page + 1) ] loop.run_until_complete(asyncio.wait(tasks))
def main(year, month): engine = get_db_engine_for_pandas() report_date = time_last_day_of_month(year=year, month=month) total_page = int( get_page_num( get_cashflow_url(report_date=report_date))['result']['pages']) loop = asyncio.get_event_loop() tasks = [ loop.create_task( parse(get_cashflow_url(report_date=report_date, page=p), engine)) for p in range(1, total_page + 1) ] loop.run_until_complete(asyncio.wait(tasks))
def main(page_size=100): engine = get_db_engine_for_pandas() total_page = math.ceil( get_page_num( get_trade_date_detail_url(psize=page_size))['data']['total'] / page_size) loop = asyncio.get_event_loop() tasks = [ loop.create_task( parse(get_trade_date_detail_url(psize=page_size, page=p), engine)) for p in range(1, total_page + 1) ] loop.run_until_complete(asyncio.wait(tasks))
def main(begin_date, end_date): engine = get_db_engine_for_pandas() total_page = int( get_page_num( get_unlocked_url( begin_date=begin_date, end_date=end_date))['pages']) loop = asyncio.get_event_loop() tasks = [ loop.create_task( parse( get_unlocked_url( begin_date=begin_date, end_date=end_date, page=p), engine)) for p in range( 1, total_page + 1)] loop.run_until_complete(asyncio.wait(tasks))