def parse_filings_on_cloud(afterdate, beforedate, get_old_active=True):
logger.info(f"Parsing filings between {afterdate} and {beforedate}.")
if get_old_active:
all_case_nums = get_all_case_nums(
afterdate, beforedate) + get_old_active_case_nums()
else:
all_case_nums = get_all_case_nums(afterdate, beforedate)
logger.info(f"Found {len(all_case_nums)} case numbers.")
parse_all_from_parse_filings(all_case_nums)
def parse_filings_on_cloud(
afterdate: datetime.date,
beforedate: datetime.date,
get_old_active=True,
showbrowser=False,
scraper: Optional[scrapers.FakeScraper] = None,
):
"""Parses filings without command line interface and outfile options."""
logger.info(f"Parsing filings between {afterdate} and {beforedate}.")
if not scraper:
scraper = scrapers.TravisScraper(headless=not showbrowser)
all_case_nums = scraper.get_all_case_nums(afterdate=afterdate,
beforedate=beforedate)
if get_old_active:
from persist import get_old_active_case_nums
all_case_nums += get_old_active_case_nums()
# using dict to eliminate duplicates
all_case_nums = list(dict.fromkeys(all_case_nums))
logger.info(
f"Found {len(all_case_nums)} case numbers (including old active ones)."
)
cases = parse_all_from_parse_filings(all_case_nums, scraper=scraper)
# persist cases only if not using the test scraper
if isinstance(scraper, scrapers.TravisScraper):
persist_parsed_cases(cases)
return cases
def parse_filings_on_cloud(afterdate: str,
beforedate: str,
get_old_active=True):
"""Same as `parse_filings()` (see below) but without command line interface and showbrowser/outfile options"""
logger.info(f"Parsing filings between {afterdate} and {beforedate}.")
if get_old_active:
all_case_nums = get_all_case_nums(
afterdate, beforedate) + get_old_active_case_nums()
else:
all_case_nums = get_all_case_nums(afterdate, beforedate)
logger.info(
f"Found {len(all_case_nums)} case numbers (including old active ones)."
)
parse_all_from_parse_filings(all_case_nums)
def test_parse_cases_from_cli(self):
ids_to_parse = ["J1-CV-20-001590"]
cases = parse_hearings.parse_all_from_parse_filings(
case_nums=ids_to_parse,
showbrowser=False,
scraper=FAKE_SCRAPER,
)
assert cases[0]["register_url"].endswith("CaseID=2286743")
assert cases[0]["hearings"][0]["hearing_type"] == "Eviction Hearing"
def parse_filings(afterdate, beforedate, outfile, showbrowser=False):
# use default firefox browser (rather than headless) is showbrowser is True
if showbrowser:
fetch_page.driver = webdriver.Chrome("./chromedriver")
all_case_nums = get_all_case_nums(afterdate,
beforedate) + get_old_active_case_nums()
parsed_cases = parse_all_from_parse_filings(all_case_nums,
showbrowser=showbrowser)
try:
json.dump(parsed_cases, outfile)
except:
logger.error("Creating the json file may have been unsuccessful.")
def parse_filings(afterdate, beforedate, outfile, showbrowser=False):
"""
Performs a full 'scraper run' between `afterdate` and `beforedate` - gets case details, events, and dispositions for all case numbers between
`afterdate` and `beforedate`. Example of date format: 9-1-2020. Also updates rows in event/disposition/case_detail table that are still active
"""
# use default chrome browser (rather than headless) is showbrowser is True
if showbrowser:
fetch_page.driver = webdriver.Chrome("./chromedriver")
all_case_nums = get_all_case_nums(afterdate,
beforedate) + get_old_active_case_nums()
parsed_cases = parse_all_from_parse_filings(all_case_nums,
showbrowser=showbrowser)
try:
json.dump(parsed_cases, outfile)
except:
logger.error("Creating the json file may have been unsuccessful.")