def prepare_lbsn_city(cls, record): """Get common attributes for records of type lbsn.City""" place_record = PlaceBaseAttrShared(record) country_guid = HF.null_check(record.country_pkey.id) sub_type = HF.null_check(record.sub_type) prepared_record = (place_record.origin_id, place_record.guid, place_record.name, place_record.name_alternatives, HF.return_ewkb_from_geotext( place_record.geom_center), HF.return_ewkb_from_geotext(place_record.geom_area), place_record.url, country_guid, sub_type) return prepared_record
def __init__(self, record=None): if record is None: record = lbsn.UserGroup() self.origin_id = record.pkey.origin.origin_id self.guid = record.pkey.id self.usergroup_name = HF.null_check(record.usergroup_name) self.usergroup_description = HF.null_check( record.usergroup_description) self.member_count = HF.null_check(record.member_count) self.usergroup_createdate = HF.null_check_datetime( record.usergroup_createdate) self.user_owner = HF.null_check(record.user_owner_pkey.id)
def __init__(self, record=None): if record is None: record = lbsn.Place() # init empty structure self.origin_id = record.pkey.origin.origin_id # = 3 self.guid = record.pkey.id self.name = HF.null_check(record.name) # because ProtoBuf Repeated Field does not support distinct rule, # we remove any duplicates in list fields prior to submission here self.name_alternatives = list(set(record.name_alternatives)) if self.name and self.name in self.name_alternatives: self.name_alternatives.remove(self.name) self.url = HF.null_check(record.url) self.geom_center = HF.null_check(record.geom_center) self.geom_area = HF.null_check(record.geom_area)
def hll_concat_latlng(record: lbsn.Post) -> str: """Concat post lat lng coordinates to string""" if record.post_geoaccuracy == 'latlng': coordinates_geom = HF.null_check(record.post_latlng) coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) return f'{coordinates.lat}:{coordinates.lng}' return '0:0'
def __init__(self, record: Union[lbsn.Post, lbsn.Place, lbsn.City, lbsn.Country] = None): super().__init__() self.key["guid"] = None self.attrs['name'] = None self.attrs['geom_center'] = None self.attrs['geom_area'] = None self.metrics['pud_hll'] = set() self.metrics['utl_hll'] = set() self.metrics['latlng_hll'] = set() if record is None: # init empty return name = None geom_area = None if isinstance(record, lbsn.Post): coordinates_geom = record.post_latlng coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) # use concat lat:lng as key of no place_key available # this should later implement assignemnt based on area # intersection self.key["guid"] = HLF.hll_concat( [coordinates.lat, coordinates.lng]) elif isinstance(record, (lbsn.Place, lbsn.City, lbsn.Country)): name = HF.null_check(record.name) coordinates_geom = record.geom_center geom_area = record.geom_area # use key from place, city or country record self.key["guid"] = HLF.hll_concat_origin_guid(record) self.attrs['name'] = name self.attrs['geom_center'] = HF.return_ewkb_from_geotext( coordinates_geom) self.attrs['geom_area'] = HF.return_ewkb_from_geotext(geom_area)
def __init__(self, relationship=None): if relationship is None: relationship = lbsn.Relationship() self.origin_id = relationship.pkey.relation_to.origin.origin_id self.guid = relationship.pkey.relation_to.id self.guid_rel = relationship.pkey.relation_from.id self.rel_type = HF.null_check( lbsn.Relationship().RelationshipType.Name( relationship.relationship_type)).lower()
def __init__(self, record=None): if record is None: record = lbsn.PostReaction() self.origin_id = record.pkey.origin.origin_id self.guid = record.pkey.id self.reaction_latlng = HF.null_geom_check(record.reaction_latlng) self.user_guid = HF.null_check(record.user_pkey.id) self.referenced_post = HF.null_check(record.referencedPost_pkey.id) self.referenced_postreaction = HF.null_check( record.referencedPostreaction_pkey.id) self.reaction_type = HF.turn_lower( HF.null_check(lbsn.PostReaction().ReactionType.Name( record.reaction_type))) self.reaction_date = HF.null_check_datetime(record.reaction_date) self.reaction_content = HF.null_check(record.reaction_content) self.reaction_like_count = HF.null_check(record.reaction_like_count) self.user_mentions = list( set([pkey.id for pkey in record.user_mentions_pkey]))
def __init__(self, record: Union[lbsn.Post, lbsn.Place] = None): super().__init__() self.key['place_guid'] = None self.attrs['geom_center'] = None self.attrs['geom_area'] = None self.attrs['name'] = None self.metrics['pud_hll'] = set() self.metrics['utl_hll'] = set() if record is None: return if isinstance(record, lbsn.Post): # Post can be of Geoaccuracy "Place" without any # actual place id assigned (e.g. Flickr Geoaccuracy level < 10) # in this case, concat lat:lng as primary key coordinates_geom = record.post_latlng if not record.place_pkey.id: coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['place_guid'] = HLF.hll_concat( [coordinates.lat, coordinates.lng]) else: self.key['place_guid'] = record.place_pkey.id # additional (optional) attributes # formatted ready for sql upsert self.attrs['geom_center'] = HF.return_ewkb_from_geotext( coordinates_geom) # geom_area not available from lbsn.Post elif isinstance(record, lbsn.Place): coordinates_geom = record.geom_center coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['place_guid'] = record.pkey.id # self.key['place_guid'] = HLF.hll_concat( # [coordinates.lat, coordinates.lng]) self.attrs['geom_center'] = HF.return_ewkb_from_geotext( coordinates_geom) self.attrs['geom_area'] = HF.return_ewkb_from_geotext( HF.null_check(record.geom_area)) self.attrs['name'] = record.name
def __init__(self, record=None): if record is None: record = lbsn.Event() self.origin_id = record.pkey.origin.origin_id self.event_guid = record.pkey.id self.name = HF.null_check(record.name) self.event_latlng = HF.null_geom_check(record.event_latlng) self.event_area = HF.null_check(record.event_area) self.event_website = HF.null_check(record.event_website) self.event_date = HF.null_check_datetime(record.event_date) self.event_date_start = HF.null_check_datetime(record.event_date_start) self.event_date_end = HF.null_check_datetime(record.event_date_end) self.duration = HF.null_check(record.duration) self.place_guid = HF.null_check(record.place_pkey.id) self.city_guid = HF.null_check(record.city_pkey.id) self.country_guid = HF.null_check(record.country_pkey.id) self.user_guid = HF.null_check(record.user_pkey.id) self.event_description = HF.null_check(record.event_description) self.event_type = HF.null_check(record.event_type) self.event_share_count = HF.null_check(record.event_share_count) self.event_like_count = HF.null_check(record.event_like_count) self.event_comment_count = HF.null_check(record.event_comment_count) self.event_views_count = HF.null_check(record.event_views_count) self.event_engage_count = HF.null_check(record.event_engage_count)
def __init__(self, record=None): if record is None: record = lbsn.Post() self.origin_id = record.pkey.origin.origin_id self.guid = record.pkey.id self.post_latlng = HF.null_geom_check(record.post_latlng) self.place_guid = HF.null_check(record.place_pkey.id) self.city_guid = HF.null_check(record.city_pkey.id) self.country_guid = HF.null_check(record.country_pkey.id) self.post_geoaccuracy = HF.turn_lower( HF.null_check(lbsn.Post().PostGeoaccuracy.Name( record.post_geoaccuracy))) self.user_guid = HF.null_check(record.user_pkey.id) self.post_create_date = HF.null_check_datetime(record.post_create_date) self.post_publish_date = HF.null_check_datetime( record.post_publish_date) self.post_body = HF.null_check(record.post_body) self.post_language = HF.null_check(record.post_language.language_short) self.user_mentions = list( set([pkey.id for pkey in record.user_mentions_pkey])) self.hashtags = list(set(record.hashtags)) self.emoji = list(set(record.emoji)) self.post_like_count = HF.null_check(record.post_like_count) self.post_comment_count = HF.null_check(record.post_comment_count) self.post_views_count = HF.null_check(record.post_views_count) self.post_title = HF.null_check(record.post_title) self.post_thumbnail_url = HF.null_check(record.post_thumbnail_url) self.post_url = HF.null_check(record.post_url) self.post_type = HF.turn_lower( HF.null_check(lbsn.Post().PostType.Name(record.post_type))) self.post_filter = HF.null_check(record.post_filter) self.post_quote_count = HF.null_check(record.post_quote_count) self.post_share_count = HF.null_check(record.post_share_count) self.input_source = HF.null_check(record.input_source) self.post_content_license = HF.null_check(record.post_content_license) # optional: self.latitude = 0 self.longitude = 0
def __init__(self, record=None): if record is None: record = lbsn.User() self.origin_id = record.pkey.origin.origin_id self.guid = record.pkey.id self.user_name = HF.null_check(record.user_name) self.user_fullname = HF.null_check(record.user_fullname) self.follows = HF.null_check(record.follows) self.followed = HF.null_check(record.followed) self.group_count = HF.null_check(record.group_count) self.biography = HF.null_check(record.biography) self.post_count = HF.null_check(record.post_count) self.url = HF.null_check(record.url) self.is_private = HF.null_check(record.is_private) self.is_available = HF.null_check(record.is_available) self.user_language = HF.null_check(record.user_language.language_short) self.user_location = HF.null_check(record.user_location) self.user_location_geom = HF.null_check(record.user_location_geom) self.liked_count = HF.null_check(record.liked_count) self.active_since = HF.null_check_datetime(record.active_since) self.profile_image_url = HF.null_check(record.profile_image_url) self.user_timezone = HF.null_check(record.user_timezone) self.user_utc_offset = HF.null_check(record.user_utc_offset) self.user_groups_member = list(set(record.user_groups_member)) self.user_groups_follows = list(set(record.user_groups_follows))
def prepare_lbsn_place(cls, record): """Get common attributes for records of type lbsn.Place""" place_record = PlaceBaseAttrShared(record) # get additional attributes specific to places city_guid = HF.null_check(record.city_pkey.id) post_count = HF.null_check(record.post_count) place_description = HF.null_check(record.place_description) place_website = HF.null_check(record.place_website) place_phone = HF.null_check(record.place_phone) address = HF.null_check(record.address) zip_code = HF.null_check(record.zip_code) attributes = HF.map_to_dict(HF.null_check(record.attributes)) checkin_count = HF.null_check(record.checkin_count) like_count = HF.null_check(record.like_count) parent_places = HF.null_check(record.parent_places) prepared_record = (place_record.origin_id, place_record.guid, place_record.name, place_record.name_alternatives, HF.return_ewkb_from_geotext( place_record.geom_center), HF.return_ewkb_from_geotext(place_record.geom_area), place_record.url, city_guid, post_count, place_description, place_website, place_phone, address, zip_code, attributes, checkin_count, like_count, parent_places) return prepared_record