def persist_parsed_cases(cases: List[Dict[str, Any]]) -> None: import persist logger.info( f"Finished making case list, now will send all {len(cases)} cases to SQL." ) failed_cases = [] for parsed_case in cases: try: persist.rest_case(parsed_case) except: try: failed_cases.append(parsed_case["case_number"]) except: logger.error( "A case failed to be parsed but it doesn't have a case number." ) if failed_cases: error_message = f"Failed to send the following case numbers to SQL:\n{', '.join(failed_cases)}" log_and_email( error_message, "Case Numbers for Which Sending to SQL Failed", error=True, ) logger.info("Finished sending cases to SQL.")
def parse_all(infile, outfile, showbrowser=False): # If showbrowser is True, use the default selenium driver if showbrowser: from selenium import webdriver fetch_page.driver = webdriver.Firefox() ids_to_parse = get_ids_to_parse(infile) parsed_cases = make_case_list(ids_to_parse) for parsed_case in parsed_cases: persist.rest_case(parsed_case) json.dump(parsed_cases, outfile)
def parse_all_from_parse_filings(case_nums, showbrowser=False): if showbrowser: from selenium import webdriver fetch_page.driver = webdriver.Chrome("./chromedriver") parsed_cases = make_case_list(case_nums) for parsed_case in parsed_cases: persist.rest_case(parsed_case) return parsed_cases
def parse_all(infile, outfile, showbrowser=False): """Same as `parse_all_from_parse_filings()` but takes in a csv of case numbers instead of a list.""" # If showbrowser is True, use the default selenium driver if showbrowser: from selenium import webdriver fetch_page.driver = webdriver.Chrome("./chromedriver") ids_to_parse = get_ids_to_parse(infile) for tries in range(5): try: parsed_cases = make_case_list(ids_to_parse) for parsed_case in parsed_cases: persist.rest_case(parsed_case) json.dump(parsed_cases, outfile) logger.info(f"Successfully parsed hearings on attempt {tries + 1}") break except Exception as e: logger.error( f"Failed to parse hearings on attempt {tries + 1}. Error message: {e}" )
def parse_all_from_parse_filings(case_nums: List[str], showbrowser=False) -> List[Dict[str, Any]]: """ Gets case details for each case number in `case_nums` and sends the data to PostgreSQL. Logs any case numbers for which getting data failed. """ if showbrowser: from selenium import webdriver fetch_page.driver = webdriver.Chrome("./chromedriver") parsed_cases = make_case_list(case_nums) logger.info( f"Finished making case list, now will send all {len(parsed_cases)} cases to SQL." ) failed_cases = [] for parsed_case in parsed_cases: try: persist.rest_case(parsed_case) except: try: failed_cases.append(parsed_case["case_number"]) except: logger.error( "A case failed to be parsed but it doesn't have a case number." ) if failed_cases: error_message = f"Failed to send the following case numbers to SQL:\n{', '.join(failed_cases)}" log_and_email(error_message, "Case Numbers for Which Sending to SQL Failed", error=True) logger.info("Finished sending cases to SQL.") return parsed_cases