class EmojiLatLngBase(hll.HllBase): """Composite Base (c-base) that extends from HLL base Class Note: To distinguish c-bases which are composite bases combining aspects from multiple facets, they're termed with a leading underscore """ NAME = hll.HllBaseRef(facet=FACET, base='_emoji_latlng') def __init__(self, record: lbsn.Post = None, emoji: str = None): super().__init__() self.key['latitude'] = None self.key['longitude'] = None self.key['emoji'] = None self.attrs['latlng_geom'] = None self.metrics['pud_hll'] = set() if emoji is None: # init empty return self.key['emoji'] = emoji if record is None: # init empty return if isinstance(record, lbsn.Post): coordinates_geom = record.post_latlng coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['latitude'] = coordinates.lat self.key['longitude'] = coordinates.lng # additional (optional) attributes # formatted ready for sql upsert self.attrs['latlng_geom'] = HF.return_ewkb_from_geotext( coordinates_geom) else: raise ValueError("Parsing of LatLngBase only supported " "from lbsn.Post")
class MonthHashtagBase(hll.HllBase): """Composite Base (c-base) that extends from HLL base Class Note: To distinguish c-bases which are composite bases combining aspects from multiple facets, they're termed with a leading underscore """ NAME = hll.HllBaseRef(facet=FACET, base='_month_hashtag') def __init__(self, record: lbsn.Post = None, hashtag: str = None): super().__init__() self.key['year'] = None self.key['month'] = None self.key['hashtag'] = None if hashtag is None: # init empty return self.key['hashtag'] = hashtag.lower() if record is None: # init empty return if isinstance(record, lbsn.Post): post_date_time = HLF.merge_dates_post(record) if post_date_time is None: return date = post_date_time.date() self.key['year'] = date.year self.key['month'] = date.month else: raise ValueError("Parsing of MonthHashtagBase only supported " "from lbsn.Post")
class LatLngBase(hll.HllBase): """Extends Base Class""" # base reference, eg. # e.g.: facet: temporal, base: date NAME = hll.HllBaseRef(facet=FACET, base='latlng') def __init__(self, record: lbsn.Post = None): super().__init__() # init key and any additional attributes self.key['latitude'] = None self.key['longitude'] = None self.attrs['latlng_geom'] = None # init additional metrics # beyond those defined inHllBase self.metrics['pud_hll'] = set() self.metrics['utl_hll'] = set() if record is None: # init empty return if isinstance(record, lbsn.Post): coordinates_geom = record.post_latlng coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['latitude'] = coordinates.lat self.key['longitude'] = coordinates.lng # additional (optional) attributes # formatted ready for sql upsert self.attrs['latlng_geom'] = HF.return_ewkb_from_geotext( coordinates_geom) else: raise ValueError("Parsing of LatLngBase only supported " "from lbsn.Post")
class CountryBase(SpatialBase): """Extends Spatial Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='country') def __init__(self, record: lbsn.Post = None): super().__init__(record) # rename key accordingly self.key["country_guid"] = self.key.pop("guid")
class DomainBase(TopicalBase): """Extends Topical Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='domain') def __init__(self, domain: str = None): super().__init__() self.key['domain'] = None if domain is None: # init empty return self.key['domain'] = domain
class TopicBase(TopicalBase): """Extends Topical Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='topic') def __init__(self, topic: Tuple[str] = None): super().__init__() self.key['topic'] = None if topic is None: # init empty return self.key['topic'] = topic
class TermBase(TopicalBase): """Extends Topical Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='term') def __init__(self, term: str = None): super().__init__() self.key['term'] = None if term is None: # init empty return self.key['term'] = term.lower()
class EmojiBase(TopicalBase): """Extends Topical Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='emoji') def __init__(self, emoji: str = None): super().__init__() self.key['emoji'] = None if emoji is None: # init empty return self.key['emoji'] = emoji
class HashtagBase(TopicalBase): """Extends Topical Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='hashtag') def __init__(self, hashtag: str = None): super().__init__() self.key['hashtag'] = None if hashtag is None: # init empty return self.key['hashtag'] = hashtag.lower()
class CultureBase(SocialBase): """Extends Social Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='culture') def __init__(self, record: lbsn.Language = None): super().__init__() self.key['id'] = None if record is None: # init empty return self.key['id'] = record.language_short
class UserBase(SocialBase): """Extends Social Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='user') def __init__(self, record: lbsn.User = None): super().__init__() self.key['user_guid'] = None if record is None: # init empty return # TODO: implement one-way-hashing self.key['user_guid'] = HLF.hll_concat_origin_guid(record)
class DayofweekBase(TemporalBase): """Extends Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='dayofweek') def __init__(self, record: lbsn.Post = None): super().__init__() self.key['weekday'] = None if record is None: # init empty return post_date_time = HLF.merge_dates_post(record) self.key['weekday'] = post_date_time.weekday
class TimestampBase(TemporalBase): """Extends Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='timestamp') def __init__(self, record: lbsn.Post = None): super().__init__() self.key["timestamp"] = None if record is None: # init empty return post_date_time = HLF.merge_dates_post(record) self.key["timestamp"] = post_date_time
class TimeofdayBase(TemporalBase): """Extends Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='timeofday') def __init__(self, record: lbsn.Post = None): super().__init__() self.key['timeofday'] = None if record is None: # init empty return post_date_time = HLF.merge_dates_post(record) # remove microseconds from datetime self.key['timeofday'] = post_date_time.time.replace(microsecond=0)
class YearBase(TemporalBase): """Extends Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='year') def __init__(self, record: lbsn.Post = None): super().__init__() self.key['year'] = None if record is None: # init empty return post_date_time = HLF.merge_dates_post(record) if post_date_time is None: return date = post_date_time.date() self.key['year'] = date.year
class CommunityBase(SocialBase): """Extends Social Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='community') def __init__(self, record: Union[lbsn.Origin, lbsn.Post] = None): super().__init__() self.attrs['name'] = None self.key['id'] = None if record is None: # init empty return if hasattr(record, "origin_id"): self.key['id'] = record.origin_id self.attrs['name'] = record.DESCRIPTOR.name else: self.key['id'] = record.pkey.origin.origin_id
class FriendsBase(SocialBase): """Extends Social Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='friends') def __init__(self, friends_record: Tuple[lbsn.User, lbsn.User] = None): super().__init__() self.key['user_guid'] = None self.key['user_guid_friend'] = None if friends_record is None: # init empty return # TODO: implement one-way-hashing self.key['user_guid'] = HLF.hll_concat_origin_guid( friends_record[0]) self.key['user_guid_friend'] = HLF.hll_concat_origin_guid( friends_record[1])
class DateBase(TemporalBase): """Extends Base Class""" NAME = hll.HllBaseRef(facet=FACET, base='date') def __init__(self, record: lbsn.Post = None): super().__init__() self.key['date'] = None self.attrs['name'] = None if record is None: # init empty return post_date_time = HLF.merge_dates_post(record) if post_date_time is None: return # optional: add name of date (e.g. "New Year's Day") self.key['date'] = post_date_time.date()
class PlaceBase(hll.HllBase): """Extends HllBase Class""" NAME = hll.HllBaseRef(facet=FACET, base='place') def __init__(self, record: Union[lbsn.Post, lbsn.Place] = None): super().__init__() self.key['place_guid'] = None self.attrs['geom_center'] = None self.attrs['geom_area'] = None self.attrs['name'] = None self.metrics['pud_hll'] = set() self.metrics['utl_hll'] = set() if record is None: return if isinstance(record, lbsn.Post): # Post can be of Geoaccuracy "Place" without any # actual place id assigned (e.g. Flickr Geoaccuracy level < 10) # in this case, concat lat:lng as primary key coordinates_geom = record.post_latlng if not record.place_pkey.id: coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['place_guid'] = HLF.hll_concat( [coordinates.lat, coordinates.lng]) else: self.key['place_guid'] = record.place_pkey.id # additional (optional) attributes # formatted ready for sql upsert self.attrs['geom_center'] = HF.return_ewkb_from_geotext( coordinates_geom) # geom_area not available from lbsn.Post elif isinstance(record, lbsn.Place): coordinates_geom = record.geom_center coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['place_guid'] = record.pkey.id # self.key['place_guid'] = HLF.hll_concat( # [coordinates.lat, coordinates.lng]) self.attrs['geom_center'] = HF.return_ewkb_from_geotext( coordinates_geom) self.attrs['geom_area'] = HF.return_ewkb_from_geotext( HF.null_check(record.geom_area)) self.attrs['name'] = record.name
class MonthLatLngBase(hll.HllBase): """Composite Base (c-base) that extends from HLL base Class Note: To distinguish c-bases which are composite bases combining aspects from multiple facets, they're termed with a leading underscore """ NAME = hll.HllBaseRef(facet=FACET, base='_month_latlng') def __init__(self, record: lbsn.Post = None, hashtag: str = None): super().__init__() self.key['year'] = None self.key['month'] = None self.key['latitude'] = None self.key['longitude'] = None self.attrs['latlng_geom'] = None if record is None: # init empty return if isinstance(record, lbsn.Post): post_date_time = HLF.merge_dates_post(record) if post_date_time is None: return date = post_date_time.date() self.key['year'] = date.year self.key['month'] = date.month coordinates_geom = record.post_latlng coordinates = HF.get_coordinates_from_ewkt(coordinates_geom) self.key['latitude'] = coordinates.lat self.key['longitude'] = coordinates.lng # additional (optional) attributes # formatted ready for sql upsert self.attrs['latlng_geom'] = HF.return_ewkb_from_geotext( coordinates_geom) else: raise ValueError("Parsing of MonthLatLngBase only supported " "from lbsn.Post")
class TemplateBase(TopicalBase): """Example Base class that extends TopicalBase Additional steps: * in shared_structure_proto_hlldb.py: extract_hll_bases(), define which lbsn.records are mapped to this class * if multiple TemplateBase can be extracted from a single lbsn record (e.g. such as multiple HashtagBase for a single lbsn.Post), update process in hll_bases.py: base_factory() * in submit_data.py, add TemplateBase to batched_hll_records so they are actually used in the mapping procedure * if you need specific metrics/measurements that are noit yet defined, update HllMetrics and get_hll_metrics() in shared_structure_proto_hlldb.py """ # define the name reference (str-identifier) for this class # any class with NAME is automatically registered NAME = hll.HllBaseRef(facet=FACET, base='topicexample') def __init__(self, record: Union[lbsn.Post, lbsn.Place] = None): """Initialize TemplateBase from lbsn record Define here from which lbsn record types this base can be initialized, example: lbsn.Post, lbsn.Place """ # initialize parent class (TopicalBase) super().__init__() # TemplateBase can also be initialized empty, # for this reason we specify the structure # before mapping record values self.key['topic_base_id'] = None self.attrs['topic_attr1'] = None self.attrs['topic_attr2'] = None self.attrs['topic_attr3'] = None # TemplateBase inherits from TopicalBase # thus, metrics latlng_hll, place_hll and pud_hll # are already defined. Specify additional metrics below # or remove inheritance and define from scratch self.metrics['utl_hll'] = set() # if no record is supplied, init empty if record is None: return # now it's time to extract (map) lbsn record to this base: # * first define how the key is extracted self.key['topic_base_id'] = self.extract_topic_base_id(record) # * afterwards define how additional (optional) attributes are # extracted self.key['topic_attr1'] = self.extract_topic_base_attrs(record) @classmethod def extract_topic_base_id(cls, record: Union[lbsn.Post, lbsn.Place]): """Template method: define how base key is extracted from record""" return @classmethod def extract_topic_base_attrs(cls, record: Union[lbsn.Post, lbsn.Place]): """Template method: define how additional attrs are extracted from record """ return