def test_state_listpage():
url = "https://crawlib.readthedocs.io/_static/state-list.html"
response = requests.get(url)
parse_result = html_parser.parse(
response=response,
child_class=State,
item_class=RdsItem,
)
parse_result.process_item(engine=engine)
def test_zipcode_listpage():
for city in City.get_all_unfinished(engine):
response = requests.get(city.build_url())
parse_result = html_parser.parse(
response=response,
parent=city,
child_class=Zipcode,
item_class=RdsItem,
)
parse_result.process_item(engine=engine)
def test_city_listpage():
for state in State.get_all_unfinished(engine):
response = requests.get(state.build_url())
parse_result = html_parser.parse(
response=response,
parent=state,
child_class=City,
item_class=RdsItem,
)
parse_result.process_item(engine=engine)