def fetch_and_parse(url):
parsed_page = None
while parsed_page == None:
page_html = get_data(url)
parsed_page = parse_data(page_html)
if parsed_page == None:
sleep = 30
print(f"try again in {sleep} seconds")
error_status = Status("faild")
error_status = error_status.create_response()
post_data('http://server:5000/api/v1/logs', error_status)
time.sleep(sleep)
return parsed_page
def main():
time.sleep(16)
print('scrawl in process')
new_main_page = fetch_and_parse(URL)
pages_links = get_links(new_main_page)
new_items = 0
for link in pages_links:
parsed_paste = fetch_and_parse(link)
new_paste = get_info(parsed_paste)
paste_obj = new_paste.create_object()
r = post_data('http://server:5000/api/v1/pastes', paste_obj)
print(r)
if r["created"] == "True":
new_items += 1
success_status = Status("success", new_items)
success_status = success_status.create_response()
post_data('http://server:5000/api/v1/logs', success_status)
print(f'scrawl finished - added {new_items} new pastes')
