class Source: def __init__(self, asId, hnId): self._asId = asId self._hnId = hnId self._ics = ICS(offset=1, regex="Erinnerung: Abfuhr (.*) morgen") def fetch(self): args = { "asId": self._asId, "hnId": self._hnId, "adresse": "MeineAdresse" } # get ics file r = requests.post( "https://www.stadtreinigung.hamburg/privatkunden/abfuhrkalender/Abfuhrtermin.ics", data=args, ) dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, pois): self.ics = ICS() self.pois = pois def fetch(self): r = requests.get( url="https://www.kwb-goslar.de/output/options.php", params={ "ModID": "48", "call": "ical", "pois": self.pois, }, headers={ "Referer": "https://www.kwb-goslar.de", }, ) if not r.ok: raise Exception(f"Error: failed to fetch url: {r.request.url}") dates = self.ics.convert(r.text) entries = [] for d in dates: date, waste_type = d icon = ICON_MAP.get(waste_type, "mdi:trash-can-outline") entries.append(Collection(date=date, t=waste_type, icon=icon)) return entries
class Source: def __init__(self, city, street): self._city = city self._street = street self._ics = ICS() def fetch(self): # fetch "Gelber Sack" args = {"g": self._city} r = requests.get( "https://was-wolfsburg.de/subgelberweihgarten/php/abfuhrgelber.php", params=args, ) entries = [] match = re.findall(r"(\d{2})\.(\d{2})\.(\d{4})", r.text) for m in match: date = datetime.date(day=int(m[0]), month=int(m[1]), year=int(m[2])) entries.append(Collection(date, "Gelber Sack")) # fetch remaining collections args = {"ortabf": self._street} r = requests.post( "https://was-wolfsburg.de/subabfuhrtermine/ics_abfuhrtermine3.php", data=args, ) dates = self._ics.convert(r.text) for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, standort): self._standort = standort self._ics = ICS() def fetch(self): now = datetime.datetime.now().date() r = requests.get( "https://stadtplan.dresden.de/project/cardo3Apps/IDU_DDStadtplan/abfall/ical.ashx", params={ "STANDORT": self._standort, "DATUM_VON": now.strftime("%d.%m.%Y"), "DATUM_BIS": (now + datetime.timedelta(days=365)).strftime("%d.%m.%Y"), }, ) dates = self._ics.convert(r.text) # example: "Leerung Gelbe Tonne, Bio-Tonne" entries = [] for d in dates: if d[1] == "Abfallkalender endet bald": continue types = d[1].removeprefix("Leerung ") for type in types.split(", "): entries.append(Collection(d[0], type)) return entries
class Source: def __init__(self, city, street): self._city = city self._street = street self._ics = ICS() def fetch(self): # retrieve list of cities r = requests.get("https://www.awr.de/api_v2/collection_dates/1/orte") cities = json.loads(r.text) # create city to id map from retrieved cities city_to_id = { city["ortsbezeichnung"]: city["ortsnummer"] for (city) in cities["orte"] } if self._city not in city_to_id: _LOGGER.error(f"city not found: {self._city}") return [] cityId = city_to_id[self._city] # retrieve list of streets r = requests.get( f"https://www.awr.de/api_v2/collection_dates/1/ort/{cityId}/strassen" ) streets = json.loads(r.text) # create street to id map from retrieved cities street_to_id = { street["strassenbezeichnung"]: street["strassennummer"] for (street) in streets["strassen"] } if self._street not in street_to_id: _LOGGER.error(f"street not found: {self._street}") return [] streetId = street_to_id[self._street] # retrieve list of waste types r = requests.get( f"https://www.awr.de/api_v2/collection_dates/1/ort/{cityId}/abfallarten" ) waste_types = json.loads(r.text) wt = "-".join([t["id"] for t in waste_types["abfallarten"]]) # get ics file r = requests.get( f"https://www.awr.de/api_v2/collection_dates/1/ort/{cityId}/strasse/{streetId}/hausnummern/0/abfallarten/{wt}/kalender.ics" ) dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, city, types, street=None): self._city = city self._street = street self._types = types self._ics = ICS() self._iconMap = { "Restmüll": "mdi:trash-can", "Grünabfall": "mdi:leaf", "Gelber Sack": "mdi:sack", "Papiertonne": "mdi:package-variant", "Bildschirm-/Kühlgeräte": "mdi:television-classic", "Schadstoffsammlung": "mdi:biohazard", "altmetalle": "mdi:nail", } def fetch(self): now = datetime.now() entries = self.fetch_year(now.year, self._city, self._street, self._types) if now.month == 12: # also get data for next year if we are already in december try: entries.extend( self.fetch_year((now.year + 1), self._city, self._street, self._types)) except Exception: # ignore if fetch for next year fails pass return entries def fetch_year(self, year, city, street, types): args = { "city": city, "street": street, "year": year, "types[]": types, "go_ics": "Download", } # get ics file r = requests.get("https://www.abfallkalender-zak.de", params=args) # parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: waste_type = d[1] next_pickup_date = d[0] entries.append( Collection(date=next_pickup_date, t=waste_type, icon=self._iconMap.get(waste_type, "mdi:trash-can"))) return entries
class Source: def __init__(self, ort, dropzone, ics_with_drop=False): self._ort = ort self._dropzone = dropzone self._ics_with_drop = ics_with_drop self._ics = ICS() def fetch(self): now = datetime.datetime.now() entries = self.fetch_year(now.year) if now.month == 12: # also get data for next year if we are already in december try: entries.extend(self.fetch_year(now.year + 1)) except Exception: # ignore if fetch for next year fails pass return entries def fetch_year(self, year): args = { "action": "execute_create_ics", "ort": self._ort, "dropzone": self._dropzone, "year": year, "ics_with_drop": "true" if self._ics_with_drop else "false", } # step 1: prepare ics file r = requests.post( "https://www.abfall-kreis-tuebingen.de/wp-admin/admin-ajax.php", data=args) # request returns a string with the format "\n\n\n\n\n\n\n\n<url>|<file>" # with url ::= https://www.abfall-kreis-tuebingen.de/wp-content/uploads/abfuhrtermine_id_XXXXXX.ics # and file ::= abfuhrtermine_id_XXXXXX.ics (url, file) = r.text.strip().split("|") # print(f"url = {url}, file = {file}") # step 2: get ics file r = requests.get(url) r.encoding = "utf-8" # requests doesn't guess the encoding correctly ics_file = r.text # step 3: delete ics file r = requests.post( "https://www.abfall-kreis-tuebingen.de/wp-admin/admin-ajax.php", data={"action": "execute_remove_ics"}, ) # parse ics file dates = self._ics.convert(ics_file) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, streetId=None, streetName=None): self._streetId = streetId self._streetName = streetName self._ics = ICS() def fetch(self): # get token if self._streetName: url = "https://wellington.govt.nz/layouts/wcc/GeneralLayout.aspx/GetRubbishCollectionStreets" data = {"partialStreetName": self._streetName} r = requests.post(url, json=data) data = json.loads(r.text) if len(data["d"]) == 0: raise Exception( f"No result found for streetName {self._streetName}") if len(data["d"]) > 1: raise Exception( f"More then one result returned for streetName {self._streetName}, be more specific or use streetId instead" ) self._streetId = data["d"][0].get("Key") if not self._streetId: raise Exception("No streetId supplied") url = "https://wellington.govt.nz/~/ical/" params = { "type": "recycling", "streetId": self._streetId, "forDate": datetime.date.today(), } r = requests.get(url, params=params) if not r.text.startswith("BEGIN:VCALENDAR"): raise Exception(f"{self._streetId} is not a valid streetID") dates = self._ics.convert(r.text) entries = [] for d in dates: for wasteType in d[1].split("&"): wasteType = wasteType.strip() entries.append( Collection( d[0], wasteType, picture=PICTURE_MAP[wasteType], icon=ICON_MAP[wasteType], )) return entries
class Source: def __init__(self, ort, strasse, hausnummer): self._ort = ort self._strasse = strasse self._hausnummer = hausnummer self._ics = ICS() def fetch(self): session = requests.session() r = session.get( "https://webudb.udb.at/WasteManagementUDB/WasteManagementServlet?SubmitAction=wasteDisposalServices&InFrameMode=TRUE" ) # add all hidden input fields to form data p = HiddenInputParser() p.feed(r.text) args = p.args args["Focus"] = "Hausnummer" args["SubmitAction"] = "forward" args["Ort"] = self._ort args["Strasse"] = self._strasse args["Hausnummer"] = self._hausnummer r = session.post( "https://webudb.udb.at/WasteManagementUDB/WasteManagementServlet", data=args) args["ApplicationName"] = "com.athos.kd.udb.AbfuhrTerminModel" args["Focus"] = None args["IsLastPage"] = "true" args["Method"] = "POST" args["PageName"] = "Terminliste" args["SubmitAction"] = "filedownload_ICAL" del args["Ort"] del args["Strasse"] del args["Hausnummer"] r = session.post( "https://webudb.udb.at/WasteManagementUDB/WasteManagementServlet", data=args) dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, district, city, street=None): self._district = district self._city = city self._street = street self._ics = ICS() def fetch(self): session = requests.Session() params = { "Abfuhrbezirk": self._district, "Ortschaft": self._city, "Strasse": self._street, } r = requests.post( "https://www.awb-lm.de/generator/abfuhrtermine.php", data=params ) r.raise_for_status() soup = BeautifulSoup(r.text, features="html.parser") downloads = soup.find_all("a", href=True) ics_url = None for download in downloads: href = download.get("href") if "cache/ical" in href: ics_url = href if ics_url is None: raise Exception(f"ics url not found") # get ics file r = session.get("https://www.awb-lm.de" + ics_url, headers=HEADERS) r.raise_for_status() # parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1].split(" am ")[0])) return entries
class Source: def __init__(self, street, house_number): self._street = street self._house_number = house_number self._ics = ICS() def fetch(self): params = { "name": self._street, } # get list of streets and house numbers r = requests.get( "https://stadtreinigung-leipzig.de/rest/wastecalendarstreets", params=params) data = json.loads(r.text) if len(data["results"]) == 0: _LOGGER.error(f"street not found: {self._street}") return [] street_entry = data["results"].get(self._street) if street_entry is None: _LOGGER.error(f"street not found: {self._street}") return [] id = street_entry.get(str(self._house_number)) if id is None: _LOGGER.error(f"house_number not found: {self._house_number}") return [] # get ics file params = { "position_nos": id, } r = requests.get( "https://stadtreinigung-leipzig.de/wir-kommen-zu-ihnen/abfallkalender/ical.ics", params=params, ) dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1].removesuffix(", "))) return entries
class Source: def __init__(self, street, house_number): self._street = street self._house_number = house_number self._ics = ICS(regex=r"(.*)\:\s*\!") def fetch(self): args = { "id": 430, "tx_citkoabfall_abfallkalender[strasse]": str(self._street).encode("utf-8"), "tx_citkoabfall_abfallkalender[hausnummer]": str(self._house_number).encode("utf-8"), "tx_citkoabfall_abfallkalender[abfallarten][0]": 61, "tx_citkoabfall_abfallkalender[abfallarten][1]": 60, "tx_citkoabfall_abfallkalender[abfallarten][2]": 59, "tx_citkoabfall_abfallkalender[abfallarten][3]": 58, "tx_citkoabfall_abfallkalender[action]": "ics", "tx_citkoabfall_abfallkalender[controller]": "FrontendIcs", } # use '%20' instead of '+' in URL # https://stackoverflow.com/questions/21823965/use-20-instead-of-for-space-in-python-query-parameters args = urllib.parse.urlencode(args, quote_via=urllib.parse.quote) # post request r = requests.get(URL, params=args) dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, city, street=None): self._city = city self._street = street self._ics = ICS() def fetch(self): session = requests.Session() params = { "city": self._city, "street": self._street, "direct": "true", } r = session.get( "https://www.awb-es.de/abfuhr/abfuhrtermine/__Abfuhrtermine.html", params=params, ) r.raise_for_status() soup = BeautifulSoup(r.text, features="html.parser") downloads = soup.find_all("a", href=True) ics_url = None for download in downloads: href = download.get("href") if "t=ics" in href: ics_url = href if ics_url is None: raise Exception(f"ics url not found") # get ics file r = session.get(ics_url, headers=HEADERS) r.raise_for_status() # parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, hnId, asId=None): self._hnId = hnId self._ics = ICS() def fetch(self): args = {"hnIds": self._hnId, "adresse": "MeineAdresse"} # get ics file r = requests.get( "https://backend.stadtreinigung.hamburg/kalender/abholtermine.ics", params=args, ) dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, url): self._url = url self._ics = ICS() def fetch(self): r = requests.get(self._url) if r.status_code != 200: _LOGGER.error("Error querying calendar data") return [] fixed_text = r.text.replace("REFRESH - INTERVAL; VALUE = ", "REFRESH-INTERVAL;VALUE=") dates = self._ics.convert(fixed_text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, city, types, street=None): self._city = city self._street = street self._types = types self._ics = ICS() def fetch(self): now = datetime.now() entries = self.fetch_year(now.year, self._city, self._street, self._types) if now.month == 12: # also get data for next year if we are already in december try: entries.extend( self.fetch_year((now.year + 1), self._city, self._street, self._types)) except Exception: # ignore if fetch for next year fails pass return entries def fetch_year(self, year, city, street, types): args = { "city": city, "street": street, "year": year, "types[]": types, "go_ics": "Download", } # get ics file r = requests.get("https://www.abfallkalender-zak.de", params=args) # parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, district: str): self._district = district self._ics = ICS() def fetch(self): now = datetime.now() entries = self._fetch_year(now.year) if now.month == 12: # also get data for next year if we are already in december with contextlib.suppress(Exception): entries.extend(self._fetch_year(now.year + 1)) return entries def _fetch_year(self, year: int): url = "/".join( str(param) for param in ( API_URL, self._district, year, REMINDER_DAY, REMINDER_HOUR, FILENAME, )) r = requests.get(url) schedule = self._ics.convert(r.text) return [ Collection(date=entry[0], t=entry[1], icon=ICON_MAP.get(entry[1], "mdi:trash-can")) for entry in schedule ]
class Source: def __init__(self, url=None, file=None, offset=None, params=None, year_field=None): self._url = url self._file = file if bool(self._url is not None) == bool(self._file is not None): raise RuntimeError("Specify either url or file") self._ics = ICS(offset) self._params = params self._year_field = year_field # replace this field in params with current year def fetch(self): if self._url is not None: if "{%Y}" in self._url or self._year_field is not None: # url contains wildcard or params contains year field now = datetime.datetime.now() # replace year in url url = self._url.replace("{%Y}", str(now.year)) # replace year in params if self._year_field is not None: if self._params is None: raise RuntimeError( "year_field specified without params") self._params[self._year_field] = str(now.year) entries = self.fetch_url(url, self._params) if now.month == 12: # also get data for next year if we are already in december url = self._url.replace("{%Y}", str(now.year + 1)) self._params[self._year_field] = str(now.year + 1) try: entries.extend(self.fetch_url(url), self._params) except Exception: # ignore if fetch for next year fails pass return entries else: return self.fetch_url(self._url, self._params) elif self._file is not None: return self.fetch_file(self._file) def fetch_url(self, url, params=None): # get ics file r = requests.get(url, params=params, headers=HEADERS) r.encoding = "utf-8" # requests doesn't guess the encoding correctly return self._convert(r.text) def fetch_file(self, file): f = open(file) return self._convert(f.read()) def _convert(self, data): dates = self._ics.convert(data) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, service, mm_frm_ort_sel=None, mm_frm_str_sel=None, mm_frm_hnr_sel=None): self._service = service self._mm_frm_ort_sel = mm_frm_ort_sel self._mm_frm_str_sel = mm_frm_str_sel self._mm_frm_hnr_sel = mm_frm_hnr_sel self._ics = ICS() def fetch(self): mm_ses = InputTextParser(name="mm_ses") url = f"https://www.muellmax.de/abfallkalender/{self._service.lower()}/res/{self._service}Start.php" r = requests.get(url) mm_ses.feed(r.text) # select "Abfuhrtermine", returns ort or an empty street search field args = {"mm_ses": mm_ses.value, "mm_aus_ort.x": 0, "mm_aus_ort.x": 0} r = requests.post(url, data=args) mm_ses.feed(r.text) if self._mm_frm_ort_sel is not None: # select city args = { "mm_ses": mm_ses.value, "xxx": 1, "mm_frm_ort_sel": self._mm_frm_ort_sel, "mm_aus_ort_submit": "weiter", } r = requests.post(url, data=args) mm_ses.feed(r.text) if self._mm_frm_str_sel is not None: # show street selection page args = { "mm_ses": mm_ses.value, "xxx": 1, "mm_frm_str_name": "", "mm_aus_str_txt_submit": "suchen", } r = requests.post(url, data=args) mm_ses.feed(r.text) # select street args = { "mm_ses": mm_ses.value, "xxx": 1, "mm_frm_str_sel": self._mm_frm_str_sel, "mm_aus_str_sel_submit": "weiter", } r = requests.post(url, data=args) mm_ses.feed(r.text) if self._mm_frm_hnr_sel is not None: # select house number args = { "mm_ses": mm_ses.value, "xxx": 1, "mm_frm_hnr_sel": self._mm_frm_hnr_sel, "mm_aus_hnr_sel_submit": "weiter", } r = requests.post(url, data=args) mm_ses.feed(r.text) # select to get ical args = { "mm_ses": mm_ses.value, "xxx": 1, "mm_ica_auswahl": "iCalendar-Datei" } r = requests.post(url, data=args) mm_ses.feed(r.text) mm_frm_fra = InputCheckboxParser(startswith="mm_frm_fra") mm_frm_fra.feed(r.text) # get ics file args = {"mm_ses": mm_ses.value, "xxx": 1, "mm_frm_type": "termine"} args.update(mm_frm_fra.value) args.update({"mm_ica_gen": "iCalendar-Datei laden"}) r = requests.post(url, data=args) mm_ses.feed(r.text) entries = [] # parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, level_1, level_2, level_3=None): self._districts = [level_1, level_2, level_3] self._ics = ICS() def fetch(self): # Use a session to keep cookies and stuff session = requests.Session() # Get the IDs of the districts on the first level # Double loading is on purpose because sometimes the webpage has an overlay # which is gone on the second try in a session r = session.get(URL, headers=HEADERS) if "Zur aufgerufenen Seite" in r.text: r = session.get(URL, headers=HEADERS) if r.status_code != 200: raise Exception(f"Error: failed to fetch first url: {URL}") # Get the IDs of the districts on the first level id = self.parse_level(r.text, 1) # Get the IDs of the districts on the second level url = ( "https://www.landkreis-harburg.de/ajax/abfall_gebiete_struktur_select.html" ) params = { "parent": id, "ebene": 1, "portal": 1, "selected_ebene": 0, } r = session.get(url, params=params, headers=HEADERS) if r.status_code != 200: raise Exception(f"Error: failed to fetch second url: {url}") # Get the IDs of the districts on the second level id = self.parse_level(r.text, 2) # Get the IDs of the third level - if applicable if self._districts[3 - 1] is not None: # Get the IDs of the districts on the third level params = { "parent": id, "ebene": 2, "portal": 1, "selected_ebene": 0, } r = session.get(url, params=params, headers=HEADERS) if r.status_code != 200: raise Exception(f"Error: failed to fetch third url: {url}") # Get the IDs of the districts on the third level id = self.parse_level(r.text, 3) # Prepare data for the real web request url = "https://www.landkreis-harburg.de/abfallkalender/abfallkalender_struktur_daten_suche.html" params = { "selected_ebene": id, "owner": 20100, } r = session.get(url, params=params, headers=HEADERS) # Sometimes there is no garbage calendar available if "Es sind keine Abfuhrbezirke hinterlegt." in r.text: raise Exception( f'Error: "Es sind keine Abfuhrbezirke hinterlegt." for "{self._districts[3-1]}". Please use different input data.' ) soup = BeautifulSoup(r.text, features="html.parser") links = soup.find_all("a") ical_url = "" for any_link in links: if " als iCal" in any_link.text: ical_url = any_link.get("href") if "ical.html" not in ical_url: raise Exception("No ical Link in the result: " + str(links)) # Get the final data r = requests.get(ical_url, headers=HEADERS) if not r.ok: raise Exception(f"Error: failed to fetch url: {ical_url}") # Parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries def parse_level(self, response, level): soup = BeautifulSoup(response, features="html.parser") select_content = soup.find_all("select", id=f"strukturEbene{level}") soup = BeautifulSoup(str(select_content), features="html.parser") options_content = soup.find_all("option") level_ids = {} for option in options_content: # Ignore the "Bitte wählen..." if option.get("value") != "0": level_ids[option.text] = option.get("value") if level_ids == {}: raise Exception(f"Error: Level {level} Dictionary empty") if self._districts[level - 1] not in level_ids: raise Exception( f"Error: District {self._districts[level]} is not in the dictionary: {level_ids}" ) return level_ids[self._districts[level - 1]]
class Source: def __init__(self, city, street=None): self._city = city self._street = street self._ics = ICS() def fetch(self): # Get the hidden parameters by loading the page session = requests.Session() r = session.get(URL) r.raise_for_status() soup = BeautifulSoup(r.text, features="html.parser") hidden_tags = soup.find_all("input", type="hidden") # Prepare data for the real web request data = {} for tag in hidden_tags: data[tag.get("name")] = tag.get("value") # Find the cities which do need a street name data_cities_with_streets = soup.find_all("input", type="text", placeholder="Ort eingeben") cities_with_streets = "" for tag in data_cities_with_streets: cities_with_streets += tag.get("data-cities-with-streets") cities_with_streets = cities_with_streets.split(",") data["tx_avlcollections_pi5[wasteCalendarLocationItem]"] = self._city data["tx_avlcollections_pi5[wasteCalendarStreetItem]"] = self._street # Remove some data which the webserver doesn't like data.pop("id", None) data.pop("tx_kesearch_pi1[page]", None) data.pop("tx_kesearch_pi1[resetFilters]", None) data.pop("tx_kesearch_pi1[sortByField]", None) data.pop("tx_kesearch_pi1[sortByDir]", None) # Depending on the city remove the street from the data set if self._city.lower() not in cities_with_streets: data.pop("tx_avlcollections_pi5[wasteCalendarStreetItem]", None) # Get the final data r = session.post(URL, data=data) r.raise_for_status() if r.text.find("Ort konnte nicht gefunden werden.") != -1: raise Exception("Error: Ort konnte nicht gefunden werden.") if r.text.find("Straße konnte nicht gefunden werden.") != -1: raise Exception("Error: Ort konnte nicht gefunden werden.") if r.text.find(".ics") == -1: raise Exception("Error: No ics link found.") soup = BeautifulSoup(r.text, features="html.parser") downloads = soup.find_all("a", href=True) ics_link = "" for download in downloads: link = download.get("href") if ".ics" in link: ics_link = link full_url = "https://www.avl-ludwigsburg.de" + ics_link return self.fetch_ics(full_url) def fetch_ics(self, url): r = requests.get(url) r.raise_for_status() # Parse ics file r.encoding = "utf-8" dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, abf_strasse, abf_hausnr): self._abf_strasse = abf_strasse self._abf_hausnr = abf_hausnr self._ics = ICS() def fetch(self): # get cookie r = requests.get("https://www.bsr.de/abfuhrkalender-20520.php") cookies = r.cookies # get street name only (without PLZ) street = self._abf_strasse.split(",")[0] # start search using string name (without PLZ) args = {"script": "dynamic_search", "step": 1, "q": street} r = requests.get( "https://www.bsr.de/abfuhrkalender_ajax.php", params=args, cookies=cookies ) # retrieve house number list args = {"script": "dynamic_search", "step": 2, "q": self._abf_strasse} r = requests.get( "https://www.bsr.de/abfuhrkalender_ajax.php", params=args, cookies=cookies ) args = { "abf_strasse": street, "abf_hausnr": self._abf_hausnr, "tab_control": "Jahr", "abf_config_weihnachtsbaeume": "", "abf_config_restmuell": "on", "abf_config_biogut": "on", "abf_config_wertstoffe": "on", "abf_config_laubtonne": "on", # "abf_selectmonth": "5 2020", # "abf_datepicker": "28.04.2020", # "listitems":7, } r = requests.post( "https://www.bsr.de/abfuhrkalender_ajax.php?script=dynamic_kalender_ajax", data=args, cookies=cookies, ) args = { "script": "dynamic_iCal_ajax", "abf_strasse": self._abf_strasse, "abf_hausnr": self._abf_hausnr, "tab_control": "Jahr", "abf_config_weihnachtsbaeume": "", "abf_config_restmuell": "on", "abf_config_biogut": "on", "abf_config_wertstoffe": "on", "abf_config_laubtonne": "on", # "abf_selectmonth": "5 2020", # "listitems":7, } # create url using private url encoding encoded = map(lambda key: f"{key}={myquote(str(args[key]))}", args.keys()) url = "https://www.bsr.de/abfuhrkalender_ajax.php?" + "&".join(encoded) r = requests.get(url, cookies=cookies) # parse ics file dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__( self, url=None, file=None, offset=None, params=None, year_field=None, method="GET", split_at=None, version=2, verify_ssl=True, ): self._url = url self._file = file if bool(self._url is not None) == bool(self._file is not None): raise RuntimeError("Specify either url or file") if version == 1: self._ics = ICS_v1(offset=offset, split_at=split_at) else: self._ics = ICS(offset=offset, split_at=split_at) self._params = params self._year_field = year_field # replace this field in params with current year self._method = method # The method to send the params self._verify_ssl = verify_ssl def fetch(self): if self._url is not None: if "{%Y}" in self._url or self._year_field is not None: # url contains wildcard or params contains year field now = datetime.datetime.now() # replace year in url url = self._url.replace("{%Y}", str(now.year)) # replace year in params if self._year_field is not None: if self._params is None: raise RuntimeError( "year_field specified without params") self._params[self._year_field] = str(now.year) entries = self.fetch_url(url, self._params) if now.month == 12: # also get data for next year if we are already in december url = self._url.replace("{%Y}", str(now.year + 1)) if self._year_field is not None: self._params[self._year_field] = str(now.year + 1) try: entries.extend(self.fetch_url(url, self._params)) except Exception: # ignore if fetch for next year fails pass return entries else: return self.fetch_url(self._url, self._params) elif self._file is not None: return self.fetch_file(self._file) def fetch_url(self, url, params=None): # get ics file if self._method == "GET": r = requests.get(url, params=params, headers=HEADERS, verify=self._verify_ssl) elif self._method == "POST": r = requests.post(url, data=params, headers=HEADERS, verify=self._verify_ssl) else: raise RuntimeError( "Error: unknown method to fetch URL, use GET or POST; got {self._method}" ) r.encoding = "utf-8" # requests doesn't guess the encoding correctly # check the return code if not r.ok: _LOGGER.error( "Error: the response is not ok; need code 200, but got code %s" % r.status_code) return [] # log the result. for debugging _LOGGER.debug(r.text) return self._convert(r.text) def fetch_file(self, file): f = open(file) return self._convert(f.read()) def _convert(self, data): dates = self._ics.convert(data) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, city, street=None): self._city = city self._street = street self._ics = ICS() def fetch(self): cityId = self.fetch_city_id(self._city) streetId = self.fetch_street_id(cityId, self._street) return self.fetch_ics(DOWNLOAD_URL.format(streetId)) def is_city_selection(self, tag, cityName): return tag['value'] != "" and tag.string == self._city def fetch_city_id(self, cityName): r = requests.get(URL) if not r.ok: raise Exception("Error: failed to fetch url: {}".format(URL)) soup = BeautifulSoup(r.text, 'html.parser') citySelection = [ a for a in soup.select('#sf_locid > option[value]') if self.is_city_selection(a, cityName) ] if len(citySelection) == 0: raise Exception( "Error: could not find id for city: '{}'".format(cityName)) if len(citySelection) > 1: raise Exception( "Error: non-unique match for city: '{}'".format(cityName)) return citySelection[0]['value'] def fetch_street_id(self, cityId, streetName): r = requests.get(AUTOCOMPLETE_URL.format(cityId, streetName), headers={"Referer": URL}) if not r.ok: raise Exception("Error: failed to fetch url: {}".format( AUTOCOMPLETE_URL.format(cityId, streetName))) streets = json.loads(r.text) if streetName != None: streetId = [item[0] for item in streets if streetName in item[1]] else: streetId = [item[0] for item in streets] if len(streetId) == 0: raise Exception( "Error: could not find streets for city id / street: {}, '{}'". format(cityId, streetName)) if len(streetId) > 1: raise Exception( "Error: non-unique match for city id / street: {}, '{}'". format(cityId, streetName)) return streetId[0] def fetch_ics(self, url): r = requests.get(url, headers={"Referer": URL}) if not r.ok: raise Exception("Error: failed to fetch url: {}".format(url)) # parse ics file r.encoding = "utf-8" dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__( self, key, f_id_kommune, f_id_strasse, f_id_bezirk=None, f_id_strasse_hnr=None, f_abfallarten=[], ): self._key = key self._kommune = f_id_kommune self._bezirk = f_id_bezirk self._strasse = f_id_strasse self._strasse_hnr = f_id_strasse_hnr self._abfallarten = f_abfallarten # list of integers self._ics = ICS() def fetch(self): # get token params = {"key": self._key, "modus": MODUS_KEY, "waction": "init"} r = requests.post("https://api.abfall.io", params=params, headers=HEADERS) # add all hidden input fields to form data # There is one hidden field which acts as a token: # It consists of a UUID key and a UUID value. p = HiddenInputParser() p.feed(r.text) args = p.args args["f_id_kommune"] = self._kommune args["f_id_strasse"] = self._strasse if self._bezirk is not None: args["f_id_bezirk"] = self._bezirk if self._strasse_hnr is not None: args["f_id_strasse_hnr"] = self._strasse_hnr for i in range(len(self._abfallarten)): args[f"f_id_abfalltyp_{i}"] = self._abfallarten[i] args["f_abfallarten_index_max"] = len(self._abfallarten) args["f_abfallarten"] = ",".join( map(lambda x: str(x), self._abfallarten)) now = datetime.datetime.now() date2 = now.replace(year=now.year + 1) args[ "f_zeitraum"] = f"{now.strftime('%Y%m%d')}-{date2.strftime('%Y%m%d')}" params = { "key": self._key, "modus": MODUS_KEY, "waction": "export_ics" } # get csv file r = requests.post("https://api.abfall.io", params=params, data=args, headers=HEADERS) # parse ics file r.encoding = "utf-8" # requests doesn't guess the encoding correctly ics_file = r.text dates = self._ics.convert(ics_file) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, street): self._street = street self._ics = ICS() def fetch(self): now = datetime.datetime.now() entries = self.fetch_year(now.year) if now.month == 12: # also get data for next year if we are already in december try: entries.extend(self.fetch_year(now.year + 1)) except Exception: # ignore if fetch for next year fails pass return entries def fetch_year(self, year): with requests.Session() as s: # get session id s.get("https://www.oberhausen.de/abfallkalender") # update session with filters and get ICS link r = s.post("https://www.oberhausen.de/abfallkalender", data={ "abfall_searchstring": self._street, "abfall_jahr": year, "actbio": "on", "actdeckgruen": "on", "actdeckrot": "on", "actdeckblau": "on", "actgelb": "on", "actpapier": "on", "submit_search": "" }) # extract ICS link parser = ICSLinkParser() parser.feed(r.text) if not "href" in parser.args or parser.args["href"] == "": raise Exception( "Error: could not extract ICS download link for year / street: {}, '{}'" .format(year, self._street)) # download ICS file r = s.get(parser.args['href']) # check the return code if not r.ok: raise Exception( "Error: the response is not ok; need code 200, but got code {}" .format(r.status_code)) # parse ics file r.encoding = "utf-8" dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries
class Source: def __init__(self, city: str, street: str, house_number: int, address_suffix: str = ""): self._city = city self._street = street self._hnr = house_number self._suffix = address_suffix self._ics = ICS() def fetch(self): session = requests.session() r = session.get( "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet", params={ "SubmitAction": "wasteDisposalServices", "InFrameMode": "TRUE" }, ) r.raise_for_status() r.encoding = "utf-8" parser = HiddenInputParser() parser.feed(r.text) args = parser.args args["Ort"] = self._city args["Strasse"] = self._street args["Hausnummer"] = str(self._hnr) args["Hausnummerzusatz"] = self._suffix args["SubmitAction"] = "CITYCHANGED" r = session.post( "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet", data=args, ) r.raise_for_status() args["SubmitAction"] = "forward" args["ContainerGewaehltRM"] = "on" args["ContainerGewaehltBM"] = "on" args["ContainerGewaehltLVP"] = "on" args["ContainerGewaehltPA"] = "on" args["ContainerGewaehltPrMuell"] = "on" r = session.post( "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet", data=args, ) r.raise_for_status() args[ "ApplicationName"] = "com.athos.kd.rheinhunsrueck.AbfuhrTerminModel" args["SubmitAction"] = "filedownload_ICAL" r = session.post( "https://aao.rh-entsorgung.de/WasteManagementRheinhunsrueck/WasteManagementServlet", data=args, ) r.raise_for_status() dates = self._ics.convert(r.text) entries = [] for d in dates: entries.append(Collection(d[0], d[1])) return entries