class EmojiLatLngBase(hll.HllBase):
"""Composite Base (c-base) that extends from HLL base Class
Note: To distinguish c-bases which are composite bases combining
aspects from multiple facets, they're termed with a leading underscore
"""
NAME = hll.HllBaseRef(facet=FACET, base='_emoji_latlng')
def __init__(self, record: lbsn.Post = None, emoji: str = None):
super().__init__()
self.key['latitude'] = None
self.key['longitude'] = None
self.key['emoji'] = None
self.attrs['latlng_geom'] = None
self.metrics['pud_hll'] = set()
if emoji is None:
# init empty
return
self.key['emoji'] = emoji
if record is None:
# init empty
return
if isinstance(record, lbsn.Post):
coordinates_geom = record.post_latlng
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['latitude'] = coordinates.lat
self.key['longitude'] = coordinates.lng
# additional (optional) attributes
# formatted ready for sql upsert
self.attrs['latlng_geom'] = HF.return_ewkb_from_geotext(
coordinates_geom)
else:
raise ValueError("Parsing of LatLngBase only supported "
"from lbsn.Post")
class MonthHashtagBase(hll.HllBase):
"""Composite Base (c-base) that extends from HLL base Class
Note: To distinguish c-bases which are composite bases combining
aspects from multiple facets, they're termed with a leading underscore
"""
NAME = hll.HllBaseRef(facet=FACET, base='_month_hashtag')
def __init__(self, record: lbsn.Post = None, hashtag: str = None):
super().__init__()
self.key['year'] = None
self.key['month'] = None
self.key['hashtag'] = None
if hashtag is None:
# init empty
return
self.key['hashtag'] = hashtag.lower()
if record is None:
# init empty
return
if isinstance(record, lbsn.Post):
post_date_time = HLF.merge_dates_post(record)
if post_date_time is None:
return
date = post_date_time.date()
self.key['year'] = date.year
self.key['month'] = date.month
else:
raise ValueError("Parsing of MonthHashtagBase only supported "
"from lbsn.Post")
class LatLngBase(hll.HllBase):
"""Extends Base Class"""
# base reference, eg.
# e.g.: facet: temporal, base: date
NAME = hll.HllBaseRef(facet=FACET, base='latlng')
def __init__(self, record: lbsn.Post = None):
super().__init__()
# init key and any additional attributes
self.key['latitude'] = None
self.key['longitude'] = None
self.attrs['latlng_geom'] = None
# init additional metrics
# beyond those defined inHllBase
self.metrics['pud_hll'] = set()
self.metrics['utl_hll'] = set()
if record is None:
# init empty
return
if isinstance(record, lbsn.Post):
coordinates_geom = record.post_latlng
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['latitude'] = coordinates.lat
self.key['longitude'] = coordinates.lng
# additional (optional) attributes
# formatted ready for sql upsert
self.attrs['latlng_geom'] = HF.return_ewkb_from_geotext(
coordinates_geom)
else:
raise ValueError("Parsing of LatLngBase only supported "
"from lbsn.Post")
class CountryBase(SpatialBase):
"""Extends Spatial Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='country')
def __init__(self, record: lbsn.Post = None):
super().__init__(record)
# rename key accordingly
self.key["country_guid"] = self.key.pop("guid")
class DomainBase(TopicalBase):
"""Extends Topical Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='domain')
def __init__(self, domain: str = None):
super().__init__()
self.key['domain'] = None
if domain is None:
# init empty
return
self.key['domain'] = domain
class TopicBase(TopicalBase):
"""Extends Topical Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='topic')
def __init__(self, topic: Tuple[str] = None):
super().__init__()
self.key['topic'] = None
if topic is None:
# init empty
return
self.key['topic'] = topic
class TermBase(TopicalBase):
"""Extends Topical Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='term')
def __init__(self, term: str = None):
super().__init__()
self.key['term'] = None
if term is None:
# init empty
return
self.key['term'] = term.lower()
class EmojiBase(TopicalBase):
"""Extends Topical Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='emoji')
def __init__(self, emoji: str = None):
super().__init__()
self.key['emoji'] = None
if emoji is None:
# init empty
return
self.key['emoji'] = emoji
class HashtagBase(TopicalBase):
"""Extends Topical Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='hashtag')
def __init__(self, hashtag: str = None):
super().__init__()
self.key['hashtag'] = None
if hashtag is None:
# init empty
return
self.key['hashtag'] = hashtag.lower()
class CultureBase(SocialBase):
"""Extends Social Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='culture')
def __init__(self, record: lbsn.Language = None):
super().__init__()
self.key['id'] = None
if record is None:
# init empty
return
self.key['id'] = record.language_short
class UserBase(SocialBase):
"""Extends Social Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='user')
def __init__(self, record: lbsn.User = None):
super().__init__()
self.key['user_guid'] = None
if record is None:
# init empty
return
# TODO: implement one-way-hashing
self.key['user_guid'] = HLF.hll_concat_origin_guid(record)
class DayofweekBase(TemporalBase):
"""Extends Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='dayofweek')
def __init__(self, record: lbsn.Post = None):
super().__init__()
self.key['weekday'] = None
if record is None:
# init empty
return
post_date_time = HLF.merge_dates_post(record)
self.key['weekday'] = post_date_time.weekday
class TimestampBase(TemporalBase):
"""Extends Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='timestamp')
def __init__(self, record: lbsn.Post = None):
super().__init__()
self.key["timestamp"] = None
if record is None:
# init empty
return
post_date_time = HLF.merge_dates_post(record)
self.key["timestamp"] = post_date_time
class TimeofdayBase(TemporalBase):
"""Extends Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='timeofday')
def __init__(self, record: lbsn.Post = None):
super().__init__()
self.key['timeofday'] = None
if record is None:
# init empty
return
post_date_time = HLF.merge_dates_post(record)
# remove microseconds from datetime
self.key['timeofday'] = post_date_time.time.replace(microsecond=0)
class YearBase(TemporalBase):
"""Extends Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='year')
def __init__(self, record: lbsn.Post = None):
super().__init__()
self.key['year'] = None
if record is None:
# init empty
return
post_date_time = HLF.merge_dates_post(record)
if post_date_time is None:
return
date = post_date_time.date()
self.key['year'] = date.year
class CommunityBase(SocialBase):
"""Extends Social Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='community')
def __init__(self, record: Union[lbsn.Origin, lbsn.Post] = None):
super().__init__()
self.attrs['name'] = None
self.key['id'] = None
if record is None:
# init empty
return
if hasattr(record, "origin_id"):
self.key['id'] = record.origin_id
self.attrs['name'] = record.DESCRIPTOR.name
else:
self.key['id'] = record.pkey.origin.origin_id
class FriendsBase(SocialBase):
"""Extends Social Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='friends')
def __init__(self, friends_record: Tuple[lbsn.User, lbsn.User] = None):
super().__init__()
self.key['user_guid'] = None
self.key['user_guid_friend'] = None
if friends_record is None:
# init empty
return
# TODO: implement one-way-hashing
self.key['user_guid'] = HLF.hll_concat_origin_guid(
friends_record[0])
self.key['user_guid_friend'] = HLF.hll_concat_origin_guid(
friends_record[1])
class DateBase(TemporalBase):
"""Extends Base Class"""
NAME = hll.HllBaseRef(facet=FACET, base='date')
def __init__(self, record: lbsn.Post = None):
super().__init__()
self.key['date'] = None
self.attrs['name'] = None
if record is None:
# init empty
return
post_date_time = HLF.merge_dates_post(record)
if post_date_time is None:
return
# optional: add name of date (e.g. "New Year's Day")
self.key['date'] = post_date_time.date()
class PlaceBase(hll.HllBase):
"""Extends HllBase Class"""
NAME = hll.HllBaseRef(facet=FACET, base='place')
def __init__(self, record: Union[lbsn.Post, lbsn.Place] = None):
super().__init__()
self.key['place_guid'] = None
self.attrs['geom_center'] = None
self.attrs['geom_area'] = None
self.attrs['name'] = None
self.metrics['pud_hll'] = set()
self.metrics['utl_hll'] = set()
if record is None:
return
if isinstance(record, lbsn.Post):
# Post can be of Geoaccuracy "Place" without any
# actual place id assigned (e.g. Flickr Geoaccuracy level < 10)
# in this case, concat lat:lng as primary key
coordinates_geom = record.post_latlng
if not record.place_pkey.id:
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['place_guid'] = HLF.hll_concat(
[coordinates.lat, coordinates.lng])
else:
self.key['place_guid'] = record.place_pkey.id
# additional (optional) attributes
# formatted ready for sql upsert
self.attrs['geom_center'] = HF.return_ewkb_from_geotext(
coordinates_geom)
# geom_area not available from lbsn.Post
elif isinstance(record, lbsn.Place):
coordinates_geom = record.geom_center
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['place_guid'] = record.pkey.id
# self.key['place_guid'] = HLF.hll_concat(
# [coordinates.lat, coordinates.lng])
self.attrs['geom_center'] = HF.return_ewkb_from_geotext(
coordinates_geom)
self.attrs['geom_area'] = HF.return_ewkb_from_geotext(
HF.null_check(record.geom_area))
self.attrs['name'] = record.name
class MonthLatLngBase(hll.HllBase):
"""Composite Base (c-base) that extends from HLL base Class
Note: To distinguish c-bases which are composite bases combining
aspects from multiple facets, they're termed with a leading underscore
"""
NAME = hll.HllBaseRef(facet=FACET, base='_month_latlng')
def __init__(self, record: lbsn.Post = None, hashtag: str = None):
super().__init__()
self.key['year'] = None
self.key['month'] = None
self.key['latitude'] = None
self.key['longitude'] = None
self.attrs['latlng_geom'] = None
if record is None:
# init empty
return
if isinstance(record, lbsn.Post):
post_date_time = HLF.merge_dates_post(record)
if post_date_time is None:
return
date = post_date_time.date()
self.key['year'] = date.year
self.key['month'] = date.month
coordinates_geom = record.post_latlng
coordinates = HF.get_coordinates_from_ewkt(coordinates_geom)
self.key['latitude'] = coordinates.lat
self.key['longitude'] = coordinates.lng
# additional (optional) attributes
# formatted ready for sql upsert
self.attrs['latlng_geom'] = HF.return_ewkb_from_geotext(
coordinates_geom)
else:
raise ValueError("Parsing of MonthLatLngBase only supported "
"from lbsn.Post")
class TemplateBase(TopicalBase):
"""Example Base class that extends TopicalBase
Additional steps:
* in shared_structure_proto_hlldb.py: extract_hll_bases(),
define which lbsn.records are mapped to this class
* if multiple TemplateBase can be extracted from a single lbsn record
(e.g. such as multiple HashtagBase for a single lbsn.Post),
update process in hll_bases.py: base_factory()
* in submit_data.py, add TemplateBase to batched_hll_records so
they are actually used in the mapping procedure
* if you need specific metrics/measurements that are noit yet defined,
update HllMetrics and get_hll_metrics()
in shared_structure_proto_hlldb.py
"""
# define the name reference (str-identifier) for this class
# any class with NAME is automatically registered
NAME = hll.HllBaseRef(facet=FACET, base='topicexample')
def __init__(self, record: Union[lbsn.Post, lbsn.Place] = None):
"""Initialize TemplateBase from lbsn record
Define here from which lbsn record types
this base can be initialized, example: lbsn.Post, lbsn.Place
"""
# initialize parent class (TopicalBase)
super().__init__()
# TemplateBase can also be initialized empty,
# for this reason we specify the structure
# before mapping record values
self.key['topic_base_id'] = None
self.attrs['topic_attr1'] = None
self.attrs['topic_attr2'] = None
self.attrs['topic_attr3'] = None
# TemplateBase inherits from TopicalBase
# thus, metrics latlng_hll, place_hll and pud_hll
# are already defined. Specify additional metrics below
# or remove inheritance and define from scratch
self.metrics['utl_hll'] = set()
# if no record is supplied, init empty
if record is None:
return
# now it's time to extract (map) lbsn record to this base:
# * first define how the key is extracted
self.key['topic_base_id'] = self.extract_topic_base_id(record)
# * afterwards define how additional (optional) attributes are
# extracted
self.key['topic_attr1'] = self.extract_topic_base_attrs(record)
@classmethod
def extract_topic_base_id(cls, record: Union[lbsn.Post, lbsn.Place]):
"""Template method: define how base key is extracted from record"""
return
@classmethod
def extract_topic_base_attrs(cls, record: Union[lbsn.Post, lbsn.Place]):
"""Template method: define how additional attrs are
extracted from record
"""
return
