def persist_parsed_cases(cases: List[Dict[str, Any]]) -> None: import persist logger.info( f"Finished making case list, now will send all {len(cases)} cases to SQL." ) failed_cases = [] for parsed_case in cases: try: persist.rest_case(parsed_case) except: try: failed_cases.append(parsed_case["case_number"]) except: logger.error( "A case failed to be parsed but it doesn't have a case number." ) if failed_cases: error_message = f"Failed to send the following case numbers to SQL:\n{', '.join(failed_cases)}" log_and_email( error_message, "Case Numbers for Which Sending to SQL Failed", error=True, ) logger.info("Finished sending cases to SQL.")
def fetch_parsed_case(case_id: str) -> Tuple[str, str]: query_result = query_case_id(case_id) if query_result is None: return None result_page, register_page = query_result result_soup = BeautifulSoup(result_page, "html.parser") register_soup = BeautifulSoup(register_page, "html.parser") register_url = hearing.get_register_url(result_soup) status, type = hearing.get_status_and_type(result_soup) if status.lower() not in hearing.statuses_map: if config.local_dev: log_and_email( f"Case {case_id} has status '{status}', which is not in our list of known statuses.", "Found Unknown Status", error=True, ) else: logger.info( f"Case {case_id} has status '{status}', which is not in our list of known statuses." ) return hearing.make_parsed_case( soup=register_soup, status=status, type=type, register_url=register_url )
def fetch_parsed_case(case_id: str) -> Tuple[str, str]: query_result = fetch_page.query_case_id(case_id) if query_result is None: return None result_page, register_page = query_result result_soup = BeautifulSoup(result_page, "html.parser") register_soup = BeautifulSoup(register_page, "html.parser") register_url = get_register_url(result_soup) status, type = get_status_and_type(result_soup) if status.lower() not in statuses_map: load_dotenv() if os.getenv("LOCAL_DEV") != "true": log_and_email( f"Case {case_id} has status '{status}', which is not in our list of known statuses.", "Found Unknown Status", error=True) else: logger.info( f"Case {case_id} has status '{status}', which is not in our list of known statuses." ) return make_parsed_case(soup=register_soup, status=status, type=type, register_url=register_url)
def active_or_inactive(status): status = status.lower() if status in statuses_map: return "Active" if statuses_map[status]["is_active"] else "Inactive" else: log_and_email( f"Can't figure out whether case with substatus '{status}' is active or inactive because '{status}' is not in our statuses map dictionary.", "Encountered Unknown Substatus", error=True) return ""
def make_case_list(ids_to_parse: List[str]) -> List[Dict[str, Any]]: """Gets case details for each case number in `ids_to_parse`""" parsed_cases, failed_ids = [], [] for id_to_parse in ids_to_parse: new_case = fetch_page.fetch_parsed_case(id_to_parse) if new_case: parsed_cases.append(new_case) else: failed_ids.append(id_to_parse) if failed_ids: error_message = f"Failed to scrape data for {len(failed_ids)} case numbers. Here they are:\n{', '.join(failed_ids)}" log_and_email(error_message, "Failed Case Numbers", error=True) return parsed_cases
def perform_task_and_catch_errors(task_function, task_name): """ Calls the function `task_function` named `task_name` (just used for logging purposes) Logs and emails error message if there is one """ before = time.time() logger.info(f"\n{task_name}...") for tries in range(1, 2): try: task_function() logger.info(f"Finished {task_name} in {round(time.time() - before, 2)} seconds.") return except Exception as error: logger.error(f"Unanticipated Error {task_name} on attempt {tries} of 1:\n{str(error)}") log_and_email(f"{task_name} failed on every attempt. Check Heroku logs for more details.", f"{task_name} failed", error=True)
def create_jpdata_df() -> DataFrame: """Creates a DataFrame with various fields to replicate the JPData2 csv on arcGIS""" def handle_null(expected_string: str) -> str: if pd.isnull(expected_string): return "" else: return expected_string def get_case_status(case: str): substatus = handle_null(case["Substatus"]).lower() if substatus in statuses_map: return statuses_map[substatus]["status"] else: log_and_email( f"Can't figure out the Status column of the JPData csv for {case['Case_Num']} because '{substatus}' is not in our statuses map dictionary.", "Encountered Unknown Substatus", error=True) return None
def parse_all_from_parse_filings(case_nums: List[str], showbrowser=False) -> List[Dict[str, Any]]: """ Gets case details for each case number in `case_nums` and sends the data to PostgreSQL. Logs any case numbers for which getting data failed. """ if showbrowser: from selenium import webdriver fetch_page.driver = webdriver.Chrome("./chromedriver") parsed_cases = make_case_list(case_nums) logger.info( f"Finished making case list, now will send all {len(parsed_cases)} cases to SQL." ) failed_cases = [] for parsed_case in parsed_cases: try: persist.rest_case(parsed_case) except: try: failed_cases.append(parsed_case["case_number"]) except: logger.error( "A case failed to be parsed but it doesn't have a case number." ) if failed_cases: error_message = f"Failed to send the following case numbers to SQL:\n{', '.join(failed_cases)}" log_and_email(error_message, "Case Numbers for Which Sending to SQL Failed", error=True) logger.info("Finished sending cases to SQL.") return parsed_cases
def update_features(layer_name: str): """Handles updating of features for JPPrecincts and JPZips layers because we can't just overwrite them (doing so ruins the joins associated with them on arcGIS)""" gis = GIS(url='https://www.arcgis.com', username=ARCGIS_USERNAME, password=ARCGIS_PASSWORD) feature_layer = gis.content.search(f"title: {layer_name}", 'Feature Layer')[0].tables[0] feature_set = feature_layer.query() all_features = [feature.as_dict for feature in feature_set.features] if layer_name == "JPZips": logger.info("Updating zip codes csv...") new_features = create_zips_df() def create_feature(zip_code, num_filings): row_with_same_zip_id = [ feature["attributes"]["ObjectId"] for feature in all_features if str(feature["attributes"]["ZIP_Code"]) == zip_code ] if row_with_same_zip_id: assert len(row_with_same_zip_id) == 1 return { "attributes": { "Number_of_Filings": num_filings, "ObjectId": row_with_same_zip_id[0] } } else: return { "attributes": { "Number_of_Filings": num_filings, "ZIP_Code": zip_code, "GEOID_Data": "8600US" + zip_code } } all_zip_codes = [ str(feature["attributes"]["ZIP_Code"]) for feature in all_features ] features_created = [ create_feature(row["ZIP_Code"], row["Number_of_Filings"]) for i, row in new_features.iterrows() ] for zip_code in all_zip_codes: if zip_code not in new_features["ZIP_Code"].tolist(): features_created.append(create_feature(zip_code, 0)) features_for_update = [ feature for feature in features_created if "ObjectId" in feature["attributes"] ] features_to_add = [ feature for feature in features_created if "ZIP_Code" in feature["attributes"] ] # get zip codes that are in our data but missing from the layer with which we're joining to_join_layer = gis.content.search( f"title: Updated_Travis_ZIPs_12022020", 'Feature Layer')[0].layers[0] to_join_features = [ feature.as_dict for feature in feature_layer.query().features ] to_join_zip_codes = [ str(feature["attributes"]["ZIP_Code"]) for feature in all_features ] zips_in_our_data_but_not_to_join = [ zip for zip in all_zip_codes if zip not in to_join_zip_codes ] if zips_in_our_data_but_not_to_join: log_and_email( f"The following zip codes are in our scraped data but not the Updated_Travis_ZIPs_12022020 arcGIS later:\n{zips_in_our_data_but_not_to_join}", "New Zip Codes") else: logger.info("Updating precincts CSV...") new_features = create_precincts_df() def create_feature(row): return { "attributes": { "Count_": row["Count"], "ObjectId": [ feature["attributes"]["ObjectId"] for feature in all_features if int(feature["attributes"]["Preceinct"]) == int( row["Precinct"]) ][0] } } features_for_update = [ create_feature(row) for i, row in new_features.iterrows() ] features_to_add = [] update_response = feature_layer.edit_features(updates=features_for_update, adds=features_to_add) update_statuses = [ result['success'] for result in update_response['updateResults'] ] add_statuses = [ result['success'] for result in update_response['addResults'] ] if all(update_statuses) and all(add_statuses): logger.info(f"Updating {layer_name} succeeded for all rows.") else: log_and_email( f"Updating {layer_name} failed for at least one row, here's the info: {update_response}", "Error Updating ArcGIS CSV", error=True)