def __init__(self, event_type=LSFEventType.normal_event, logging=False):
self.fetcher = LSFFetcher()
self._event_type = event_type
self.db_access = LSFEventDBAccess()
self._logging = logging
class LSFEventScraper:
"""
An interface for the whole module. You can scrape all events from HTW-Berlin.de and store them
to a database.
"""
_events_normal = []
_events_cancelled = []
_event_type = LSFEventType.normal_event
_overview_url = "https://lsf.htw-berlin.de/qisserver/rds?state=hiscalendar&type=0&asi="
def __init__(self, event_type=LSFEventType.normal_event, logging=False):
self.fetcher = LSFFetcher()
self._event_type = event_type
self.db_access = LSFEventDBAccess()
self._logging = logging
def set_to_cancelled_events(self):
"""
This isn't finished right now. Don't use it. It will set the scraper in scrape-cancelled-events-mode,
which will scrape all cancelled events.
"""
self._event_type = LSFEventType.cancelled_event
def set_to_normal_event(self):
"""
Sets the scraper in normal-mode. The scraper will scrape all happening events.
"""
self._event_type = LSFEventType.normal_event
def crawl_day_pages_and_save_to_disk(self):
"""
Crawls all day-overviews and stores them as html files to ./data_events.
:return:
"""
self._check_folder('data_events/')
self._check_folder('data_cancels/')
self._fetch_sites()
self.fetcher.fetch_event_sites(self._save_to_disk)
def scrape_events(self):
"""
Scrapes all events and stores them in memory.
:return:
"""
self._fetch_sites()
self.fetcher.fetch_event_sites(self._receive_event_data)
def scrape_local_sites(self):
"""
Scrapes all events from the previously fetched html files in ./data_events, and stores them in memory.
:return:
"""
self.fetcher.fetch_local_sites(self._receive_event_data, event_type=self._event_type)
def save_events_to_db(self):
"""
Stores all events in memory to the database, definied in db_credentials_PSQL.json
:return:
"""
if self._event_type is LSFEventType.normal_event:
self.db_access.add_events(self._events_normal)
else:
self.db_access.add_cancel_notes(self._events_cancelled)