def persist_parsed_cases(cases: List[Dict[str, Any]]) -> None:
import persist
logger.info(
f"Finished making case list, now will send all {len(cases)} cases to SQL."
)
failed_cases = []
for parsed_case in cases:
try:
persist.rest_case(parsed_case)
except:
try:
failed_cases.append(parsed_case["case_number"])
except:
logger.error(
"A case failed to be parsed but it doesn't have a case number."
)
if failed_cases:
error_message = f"Failed to send the following case numbers to SQL:\n{', '.join(failed_cases)}"
log_and_email(
error_message,
"Case Numbers for Which Sending to SQL Failed",
error=True,
)
logger.info("Finished sending cases to SQL.")
def parse_all(infile, outfile, showbrowser=False):
# If showbrowser is True, use the default selenium driver
if showbrowser:
from selenium import webdriver
fetch_page.driver = webdriver.Firefox()
ids_to_parse = get_ids_to_parse(infile)
parsed_cases = make_case_list(ids_to_parse)
for parsed_case in parsed_cases:
persist.rest_case(parsed_case)
json.dump(parsed_cases, outfile)
def parse_all_from_parse_filings(case_nums, showbrowser=False):
if showbrowser:
from selenium import webdriver
fetch_page.driver = webdriver.Chrome("./chromedriver")
parsed_cases = make_case_list(case_nums)
for parsed_case in parsed_cases:
persist.rest_case(parsed_case)
return parsed_cases
def parse_all(infile, outfile, showbrowser=False):
"""Same as `parse_all_from_parse_filings()` but takes in a csv of case numbers instead of a list."""
# If showbrowser is True, use the default selenium driver
if showbrowser:
from selenium import webdriver
fetch_page.driver = webdriver.Chrome("./chromedriver")
ids_to_parse = get_ids_to_parse(infile)
for tries in range(5):
try:
parsed_cases = make_case_list(ids_to_parse)
for parsed_case in parsed_cases:
persist.rest_case(parsed_case)
json.dump(parsed_cases, outfile)
logger.info(f"Successfully parsed hearings on attempt {tries + 1}")
break
except Exception as e:
logger.error(
f"Failed to parse hearings on attempt {tries + 1}. Error message: {e}"
)
def parse_all_from_parse_filings(case_nums: List[str],
showbrowser=False) -> List[Dict[str, Any]]:
"""
Gets case details for each case number in `case_nums` and sends the data to PostgreSQL.
Logs any case numbers for which getting data failed.
"""
if showbrowser:
from selenium import webdriver
fetch_page.driver = webdriver.Chrome("./chromedriver")
parsed_cases = make_case_list(case_nums)
logger.info(
f"Finished making case list, now will send all {len(parsed_cases)} cases to SQL."
)
failed_cases = []
for parsed_case in parsed_cases:
try:
persist.rest_case(parsed_case)
except:
try:
failed_cases.append(parsed_case["case_number"])
except:
logger.error(
"A case failed to be parsed but it doesn't have a case number."
)
if failed_cases:
error_message = f"Failed to send the following case numbers to SQL:\n{', '.join(failed_cases)}"
log_and_email(error_message,
"Case Numbers for Which Sending to SQL Failed",
error=True)
logger.info("Finished sending cases to SQL.")
return parsed_cases
