Example #1
0
def scrape_by_date(date_str):
    events = scrape_date(date_str)

    conn = sqlite3.connect(sqlite_file)
    c = conn.cursor()

    for event in events:

        try:

            insert_cso_into_db(c, event)

        except Exception as e:
            print('Exception: %s' % e.message)

    conn.commit()
    conn.close()
Example #2
0
def scrape_by_date(date_str):
    events = scrape_date(date_str)

    conn = sqlite3.connect(sqlite_file)
    c = conn.cursor()

    for event in events:

        try:

            insert_cso_into_db(c, event)

        except Exception as e:
            print('Exception: %s' % e.message)

    conn.commit()
    conn.close()
Example #3
0
def scrape_for_range(start_date_str, end_date_str):
    logging.info("Scraping from %s to %s" % (start_date_str, end_date_str))

    iter_date = datetime.datetime.strptime(start_date_str, "%m/%d/%Y")
    end = datetime.datetime.strptime(end_date_str, "%m/%d/%Y")

    conn = sqlite3.connect(sqlite_file)
    c = conn.cursor()

    while iter_date <= end:
        date_str = iter_date.strftime("%m/%d/%Y")
        events = scrape_date(date_str)

        for event in events:

            try:
                # Check to see if there are overlapping times.  This should not be needed, but there are
                # several inconsistencies on the MWRD webpages
                overlapping = find_overlapping_times(c, event)
                if len(overlapping) > 0:
                    for o_event in overlapping:
                        if o_event.start == event.start and o_event.stop == event.stop:
                            continue
                        logging.error(
                            "Found overlapping overflows.  Original event is: %s\n.  Overlapping event: %s" % (
                                event.to_string(), o_event.to_string()))
                        delete_from_db(c, o_event)

                        insert_cso_into_db(c, event)
                else:
                    insert_cso_into_db(c, event)

            except Exception as e:
                logging.exception(e)

        iter_date += datetime.timedelta(days=1)

    conn.commit()
    conn.close()
Example #4
0
def scrape_for_range(start_date_str, end_date_str):
    logging.info("Scraping from %s to %s" % (start_date_str, end_date_str))

    iter_date = datetime.datetime.strptime(start_date_str, "%m/%d/%Y")
    end = datetime.datetime.strptime(end_date_str, "%m/%d/%Y")

    conn = sqlite3.connect(sqlite_file)
    c = conn.cursor()

    while iter_date <= end:
        date_str = iter_date.strftime("%m/%d/%Y")
        events = scrape_date(date_str)

        for event in events:

            try:
                # Check to see if there are overlapping times.  This should not be needed, but there are
                # several inconsistencies on the MWRD webpages
                overlapping = find_overlapping_times(c, event)
                if len(overlapping) > 0:
                    for o_event in overlapping:
                        if o_event.start == event.start and o_event.stop == event.stop:
                            continue
                        logging.error(
                            "Found overlapping overflows.  Original event is: %s\n.  Overlapping event: %s"
                            % (event.to_string(), o_event.to_string()))
                        delete_from_db(c, o_event)

                        insert_cso_into_db(c, event)
                else:
                    insert_cso_into_db(c, event)

            except Exception as e:
                logging.exception(e)

        iter_date += datetime.timedelta(days=1)

    conn.commit()
    conn.close()