def _import_organizer(self, item: dict) -> int:
organizer_item = None
if "organizer" in item:
organizer_item = item["organizer"]
if not organizer_item and "author" in item:
organizer_item = item["author"]
if (organizer_item and isinstance(organizer_item, list)
and len(organizer_item) > 0 and organizer_item[0]):
organizer_item = organizer_item[0]
organizer_name = organizer_item["name"]
organizer = dict()
organizer["name"] = organizer_name
if "url" in organizer_item and self._is_url(organizer_item["url"]):
organizer["url"] = organizer_item["url"]
if "email" in organizer_item:
organizer["email"] = organizer_item["email"]
if "telephone" in organizer_item:
organizer["phone"] = organizer_item["telephone"]
if "faxNumber" in organizer_item:
organizer["fax"] = organizer_item["faxNumber"]
if "address" in organizer_item:
organizer["location"] = self._import_location(
organizer_item["address"])
hash_key = organizer_name
organizer_hash = self._hash_dict(organizer)
self.organizer_hashes.add(organizer_hash)
if hash_key in self.stored_organizer_mapping:
organizer_id = int(self.stored_organizer_mapping[hash_key])
logger.debug(
f"Found organizer {organizer_name} in mapping: {organizer_id}."
)
stored_organizer_hash = self.stored_organizer_hashes.get(
hash_key, None)
if organizer_hash == stored_organizer_hash:
logger.debug("Organizer did not change. Nothing to do.")
else:
logger.debug("Organizer did change. Updating..")
self.api_client.update_organizer(organizer_id, organizer)
r.hset("organizer_hashes", hash_key, organizer_hash)
else:
logger.debug(
f"Organizer {organizer_name} not in mapping. Inserting..")
organizer_id = self.api_client.upsert_organizer(organizer)
r.hset("organizer_mapping", hash_key, organizer_id)
r.hset("organizer_hashes", hash_key, organizer_hash)
return organizer_id
def _load_token(self) -> dict:
token = r.hgetall("token")
if "access_token" not in token:
from_env = {
"access_token": os.getenv("ACCESS_TOKEN"),
"refresh_token": os.getenv("REFRESH_TOKEN"),
"expires_in": -1,
"scope": self.scope,
"token_type": "Bearer",
}
r.hset("token", mapping=from_env)
token = r.hgetall("token")
if "expires_in" in token:
token["expires_in"] = int(token["expires_in"])
if "expires_at" in token:
token["expires_at"] = int(token["expires_at"])
return token
def _update_token(token, refresh_token=None, access_token=None):
if token.get("refresh_token", None):
r.hset("token", "refresh_token", token.get("refresh_token"))
if token.get("access_token", None):
r.hset("token", "access_token", token.get("access_token"))
if token.get("expires_at", None):
r.hset("token", "expires_at", token.get("expires_at"))
r.hdel("token", "expires_in")
def _import_place(self, item: dict) -> int:
place_item = item["location"][0]
place_name = place_item["name"]
place = dict()
place["name"] = place_name
location = dict()
if "address" in place_item:
location = self._import_location(place_item["address"])
if "coordinate" in item:
lat_str, lon_str = item["coordinate"].split(",")
latitude = float(lat_str)
longitude = float(lon_str)
if latitude != 0 and longitude != 0:
location["latitude"] = latitude
location["longitude"] = longitude
place["location"] = location
hash_key = place_name
place_hash = self._hash_dict(place)
self.place_hashes.add(place_hash)
if hash_key in self.stored_place_mapping:
place_id = int(self.stored_place_mapping[hash_key])
logger.debug(f"Found place {place_name} in mapping: {place_id}.")
stored_place_hash = self.stored_place_hashes.get(hash_key, None)
if place_hash == stored_place_hash:
logger.debug("Place did not change. Nothing to do.")
else:
logger.debug("Place did change. Updating..")
self.api_client.update_place(place_id, place)
r.hset("place_hashes", hash_key, place_hash)
else:
logger.debug(f"Place {place_name} not in mapping. Inserting..")
place_id = self.api_client.upsert_place(place)
r.hset("place_mapping", hash_key, place_id)
r.hset("place_hashes", hash_key, place_hash)
return place_id
def _import_event_from_url(self, url: str) -> int:
self.urls_in_run.add(url)
# Load event
item = self.harzinfo_loader.load_event(url)
if not item:
logger.warn("No event data.")
r.hset("url_mapping", url, "nodata")
return 0
if not self._is_url(item["url"]):
logger.warn("Invalid url.")
r.hset("url_mapping", url, "invalidurl")
return 0
# Check for duplicates
uid = item["identifier"][0]
if uid in self.uids_in_run:
logger.warn(f"Duplicate UID {uid}")
return 0
self.uids_in_run.add(uid)
# Compare to stored hashes
item_hash = self._hash_dict(item)
event_id = 0
if uid in self.stored_event_mapping:
event_id = int(self.stored_event_mapping[uid])
logger.debug(f"Found event for uid {uid} in mapping: {event_id}.")
stored_event_hash = self.stored_event_hashes.get(uid, None)
if item_hash == stored_event_hash:
logger.debug("Event did not change. Nothing to do.")
self.unchanged_event_count = self.unchanged_event_count + 1
return event_id
# Organizer
organizer_id = self._import_organizer(item)
# Place
place_id = self._import_place(item)
# Event
event = dict()
event["external_link"] = item["url"]
event["name"] = item["name"]
event["start"] = item["startDate"]
event["place"] = {"id": place_id}
event["organizer"] = {"id": organizer_id}
if "description" in item:
event["description"] = item["description"]
self._import_event_photo(event, item)
self._import_event_status(event, item)
self._add_categories(event, item)
self._add_tags(event, item)
if event_id > 0:
logger.debug("Event did change. Updating..")
self.api_client.update_event(event_id, event)
r.hset("event_hashes", uid, item_hash)
self.updated_event_count = self.updated_event_count + 1
else:
logger.debug(f"Event for uid {uid} not in mapping. Inserting..")
event_id = self.api_client.insert_event(event)
r.hset("event_mapping", uid, event_id)
r.hset("event_hashes", uid, item_hash)
r.hset("url_mapping", url, uid)
self.new_event_count = self.new_event_count + 1
return event_id