try:
for article in set(on_page_link_list + pop_top_5_links):
article, is_pop, pop_rank, on_page = parserfunctions.assess_article(
article, on_page_link_list, pop_top_5_links)
insert_statements.append([
article, pubshort, curr_time, curr_time_utc, is_pop, pop_rank,
on_page
])
except:
message = "Failed to prepare list of insert statements"
seriousness = 1
parserfunctions.error_log_entry(cur, conn, mysql_log_name, curr_time,
pubshort, homepage, seriousness,
message)
try:
parserfunctions.data_point_multi_entry(insert_statements, cur, conn,
mysql_table_name)
except:
message = "Failed to insert the statements into the database"
seriousness = 1
parserfunctions.error_log_entry(cur, conn, mysql_log_name, curr_time,
pubshort, homepage, seriousness,
message)
### Move files if necessary
if move_on_success is not None:
parserfunctions.move_file(homepage, success_dir)
### Perform closing actions
print("Finished parsing")
on_page_link_list = parserfunctions.linklist_actions(link, on_page_link_list)
except:
message = "Failed to clean up links for list of all links on page"
seriousness = 2
parserfunctions.error_log_entry(cur, conn, mysql_log_name, curr_time, pubshort, homepage, seriousness, message)
#continue
insert_statements = []
###### Prepare insert statement for each link
try:
for article in set(on_page_link_list + pop_top_5_links + pro_top_5_links):
article, is_pro, pro_rank, is_pop, pop_rank, on_page = parserfunctions.assess_article(article, on_page_link_list, pop_top_5_links, pro_top_5_links)
insert_statements.append([article, pubshort, curr_time, curr_time_utc, is_pro, pro_rank, is_pop, pop_rank, on_page])
except:
message = "Failed to prepare list of insert statements"
seriousness = 1
parserfunctions.error_log_entry(cur, conn, mysql_log_name, curr_time, pubshort, homepage, seriousness, message)
try:
parserfunctions.data_point_multi_entry(insert_statements, cur, conn, mysql_table_name)
except:
message = "Failed to insert the statements into the database"
seriousness = 1
parserfunctions.error_log_entry(cur, conn, mysql_log_name, curr_time, pubshort, homepage, seriousness, message)
### Move files if necessary
if move_on_success is not None:
parserfunctions.move_file(homepage, success_dir)
### Perform closing actions
print("Finished parsing")
