def prepare_lbsn_city(cls, record):
"""Get common attributes for records of type lbsn.City"""
place_record = PlaceBaseAttrShared(record)
country_guid = HF.null_check(record.country_pkey.id)
sub_type = HF.null_check(record.sub_type)
prepared_record = (place_record.origin_id, place_record.guid,
place_record.name, place_record.name_alternatives,
HF.return_ewkb_from_geotext(
place_record.geom_center),
HF.return_ewkb_from_geotext(place_record.geom_area),
place_record.url, country_guid, sub_type)
return prepared_record
def __init__(self, record=None):
if record is None:
record = lbsn.UserGroup()
self.origin_id = record.pkey.origin.origin_id
self.guid = record.pkey.id
self.usergroup_name = HF.null_check(record.usergroup_name)
self.usergroup_description = HF.null_check(
record.usergroup_description)
self.member_count = HF.null_check(record.member_count)
self.usergroup_createdate = HF.null_check_datetime(
record.usergroup_createdate)
self.user_owner = HF.null_check(record.user_owner_pkey.id)
def __init__(self, record=None):
if record is None:
record = lbsn.Place() # init empty structure
self.origin_id = record.pkey.origin.origin_id # = 3
self.guid = record.pkey.id
self.name = HF.null_check(record.name)
# because ProtoBuf Repeated Field does not support distinct rule,
# we remove any duplicates in list fields prior to submission here
self.name_alternatives = list(set(record.name_alternatives))
if self.name and self.name in self.name_alternatives:
self.name_alternatives.remove(self.name)
self.url = HF.null_check(record.url)
self.geom_center = HF.null_check(record.geom_center)
self.geom_area = HF.null_check(record.geom_area)
def hll_concat_latlng(record: lbsn.Post) -> str:
"""Concat post lat lng coordinates to string"""
if record.post_geoaccuracy == 'latlng':
coordinates_geom = HF.null_check(record.post_latlng)
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
return f'{coordinates.lat}:{coordinates.lng}'
return '0:0'
def __init__(self,
record: Union[lbsn.Post, lbsn.Place, lbsn.City,
lbsn.Country] = None):
super().__init__()
self.key["guid"] = None
self.attrs['name'] = None
self.attrs['geom_center'] = None
self.attrs['geom_area'] = None
self.metrics['pud_hll'] = set()
self.metrics['utl_hll'] = set()
self.metrics['latlng_hll'] = set()
if record is None:
# init empty
return
name = None
geom_area = None
if isinstance(record, lbsn.Post):
coordinates_geom = record.post_latlng
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
# use concat lat:lng as key of no place_key available
# this should later implement assignemnt based on area
# intersection
self.key["guid"] = HLF.hll_concat(
[coordinates.lat, coordinates.lng])
elif isinstance(record, (lbsn.Place, lbsn.City, lbsn.Country)):
name = HF.null_check(record.name)
coordinates_geom = record.geom_center
geom_area = record.geom_area
# use key from place, city or country record
self.key["guid"] = HLF.hll_concat_origin_guid(record)
self.attrs['name'] = name
self.attrs['geom_center'] = HF.return_ewkb_from_geotext(
coordinates_geom)
self.attrs['geom_area'] = HF.return_ewkb_from_geotext(geom_area)
def __init__(self, relationship=None):
if relationship is None:
relationship = lbsn.Relationship()
self.origin_id = relationship.pkey.relation_to.origin.origin_id
self.guid = relationship.pkey.relation_to.id
self.guid_rel = relationship.pkey.relation_from.id
self.rel_type = HF.null_check(
lbsn.Relationship().RelationshipType.Name(
relationship.relationship_type)).lower()
def __init__(self, record=None):
if record is None:
record = lbsn.PostReaction()
self.origin_id = record.pkey.origin.origin_id
self.guid = record.pkey.id
self.reaction_latlng = HF.null_geom_check(record.reaction_latlng)
self.user_guid = HF.null_check(record.user_pkey.id)
self.referenced_post = HF.null_check(record.referencedPost_pkey.id)
self.referenced_postreaction = HF.null_check(
record.referencedPostreaction_pkey.id)
self.reaction_type = HF.turn_lower(
HF.null_check(lbsn.PostReaction().ReactionType.Name(
record.reaction_type)))
self.reaction_date = HF.null_check_datetime(record.reaction_date)
self.reaction_content = HF.null_check(record.reaction_content)
self.reaction_like_count = HF.null_check(record.reaction_like_count)
self.user_mentions = list(
set([pkey.id for pkey in record.user_mentions_pkey]))
def __init__(self, record: Union[lbsn.Post, lbsn.Place] = None):
super().__init__()
self.key['place_guid'] = None
self.attrs['geom_center'] = None
self.attrs['geom_area'] = None
self.attrs['name'] = None
self.metrics['pud_hll'] = set()
self.metrics['utl_hll'] = set()
if record is None:
return
if isinstance(record, lbsn.Post):
# Post can be of Geoaccuracy "Place" without any
# actual place id assigned (e.g. Flickr Geoaccuracy level < 10)
# in this case, concat lat:lng as primary key
coordinates_geom = record.post_latlng
if not record.place_pkey.id:
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['place_guid'] = HLF.hll_concat(
[coordinates.lat, coordinates.lng])
else:
self.key['place_guid'] = record.place_pkey.id
# additional (optional) attributes
# formatted ready for sql upsert
self.attrs['geom_center'] = HF.return_ewkb_from_geotext(
coordinates_geom)
# geom_area not available from lbsn.Post
elif isinstance(record, lbsn.Place):
coordinates_geom = record.geom_center
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['place_guid'] = record.pkey.id
# self.key['place_guid'] = HLF.hll_concat(
# [coordinates.lat, coordinates.lng])
self.attrs['geom_center'] = HF.return_ewkb_from_geotext(
coordinates_geom)
self.attrs['geom_area'] = HF.return_ewkb_from_geotext(
HF.null_check(record.geom_area))
self.attrs['name'] = record.name
def __init__(self, record=None):
if record is None:
record = lbsn.Event()
self.origin_id = record.pkey.origin.origin_id
self.event_guid = record.pkey.id
self.name = HF.null_check(record.name)
self.event_latlng = HF.null_geom_check(record.event_latlng)
self.event_area = HF.null_check(record.event_area)
self.event_website = HF.null_check(record.event_website)
self.event_date = HF.null_check_datetime(record.event_date)
self.event_date_start = HF.null_check_datetime(record.event_date_start)
self.event_date_end = HF.null_check_datetime(record.event_date_end)
self.duration = HF.null_check(record.duration)
self.place_guid = HF.null_check(record.place_pkey.id)
self.city_guid = HF.null_check(record.city_pkey.id)
self.country_guid = HF.null_check(record.country_pkey.id)
self.user_guid = HF.null_check(record.user_pkey.id)
self.event_description = HF.null_check(record.event_description)
self.event_type = HF.null_check(record.event_type)
self.event_share_count = HF.null_check(record.event_share_count)
self.event_like_count = HF.null_check(record.event_like_count)
self.event_comment_count = HF.null_check(record.event_comment_count)
self.event_views_count = HF.null_check(record.event_views_count)
self.event_engage_count = HF.null_check(record.event_engage_count)
def __init__(self, record=None):
if record is None:
record = lbsn.Post()
self.origin_id = record.pkey.origin.origin_id
self.guid = record.pkey.id
self.post_latlng = HF.null_geom_check(record.post_latlng)
self.place_guid = HF.null_check(record.place_pkey.id)
self.city_guid = HF.null_check(record.city_pkey.id)
self.country_guid = HF.null_check(record.country_pkey.id)
self.post_geoaccuracy = HF.turn_lower(
HF.null_check(lbsn.Post().PostGeoaccuracy.Name(
record.post_geoaccuracy)))
self.user_guid = HF.null_check(record.user_pkey.id)
self.post_create_date = HF.null_check_datetime(record.post_create_date)
self.post_publish_date = HF.null_check_datetime(
record.post_publish_date)
self.post_body = HF.null_check(record.post_body)
self.post_language = HF.null_check(record.post_language.language_short)
self.user_mentions = list(
set([pkey.id for pkey in record.user_mentions_pkey]))
self.hashtags = list(set(record.hashtags))
self.emoji = list(set(record.emoji))
self.post_like_count = HF.null_check(record.post_like_count)
self.post_comment_count = HF.null_check(record.post_comment_count)
self.post_views_count = HF.null_check(record.post_views_count)
self.post_title = HF.null_check(record.post_title)
self.post_thumbnail_url = HF.null_check(record.post_thumbnail_url)
self.post_url = HF.null_check(record.post_url)
self.post_type = HF.turn_lower(
HF.null_check(lbsn.Post().PostType.Name(record.post_type)))
self.post_filter = HF.null_check(record.post_filter)
self.post_quote_count = HF.null_check(record.post_quote_count)
self.post_share_count = HF.null_check(record.post_share_count)
self.input_source = HF.null_check(record.input_source)
self.post_content_license = HF.null_check(record.post_content_license)
# optional:
self.latitude = 0
self.longitude = 0
def __init__(self, record=None):
if record is None:
record = lbsn.User()
self.origin_id = record.pkey.origin.origin_id
self.guid = record.pkey.id
self.user_name = HF.null_check(record.user_name)
self.user_fullname = HF.null_check(record.user_fullname)
self.follows = HF.null_check(record.follows)
self.followed = HF.null_check(record.followed)
self.group_count = HF.null_check(record.group_count)
self.biography = HF.null_check(record.biography)
self.post_count = HF.null_check(record.post_count)
self.url = HF.null_check(record.url)
self.is_private = HF.null_check(record.is_private)
self.is_available = HF.null_check(record.is_available)
self.user_language = HF.null_check(record.user_language.language_short)
self.user_location = HF.null_check(record.user_location)
self.user_location_geom = HF.null_check(record.user_location_geom)
self.liked_count = HF.null_check(record.liked_count)
self.active_since = HF.null_check_datetime(record.active_since)
self.profile_image_url = HF.null_check(record.profile_image_url)
self.user_timezone = HF.null_check(record.user_timezone)
self.user_utc_offset = HF.null_check(record.user_utc_offset)
self.user_groups_member = list(set(record.user_groups_member))
self.user_groups_follows = list(set(record.user_groups_follows))
def prepare_lbsn_place(cls, record):
"""Get common attributes for records of type lbsn.Place"""
place_record = PlaceBaseAttrShared(record)
# get additional attributes specific to places
city_guid = HF.null_check(record.city_pkey.id)
post_count = HF.null_check(record.post_count)
place_description = HF.null_check(record.place_description)
place_website = HF.null_check(record.place_website)
place_phone = HF.null_check(record.place_phone)
address = HF.null_check(record.address)
zip_code = HF.null_check(record.zip_code)
attributes = HF.map_to_dict(HF.null_check(record.attributes))
checkin_count = HF.null_check(record.checkin_count)
like_count = HF.null_check(record.like_count)
parent_places = HF.null_check(record.parent_places)
prepared_record = (place_record.origin_id, place_record.guid,
place_record.name, place_record.name_alternatives,
HF.return_ewkb_from_geotext(
place_record.geom_center),
HF.return_ewkb_from_geotext(place_record.geom_area),
place_record.url, city_guid, post_count,
place_description, place_website, place_phone,
address, zip_code, attributes, checkin_count,
like_count, parent_places)
return prepared_record