def dict_type_switcher(desc_name):
""" Create protoBuf messages by name"""
dict_switcher = {
lbsn.Country().DESCRIPTOR.name: lbsn.Country(),
lbsn.City().DESCRIPTOR.name: lbsn.City(),
lbsn.Place().DESCRIPTOR.name: lbsn.Place(),
lbsn.User().DESCRIPTOR.name: lbsn.User(),
lbsn.UserGroup().DESCRIPTOR.name: lbsn.UserGroup(),
lbsn.Post().DESCRIPTOR.name: lbsn.Post(),
lbsn.PostReaction().DESCRIPTOR.name: lbsn.PostReaction(),
lbsn.Relationship().DESCRIPTOR.name: lbsn.Relationship()
}
return dict_switcher.get(desc_name)
def __init__(self, record=None):
if record is None:
record = lbsn.PostReaction()
self.origin_id = record.pkey.origin.origin_id
self.guid = record.pkey.id
self.reaction_latlng = HF.null_geom_check(record.reaction_latlng)
self.user_guid = HF.null_check(record.user_pkey.id)
self.referenced_post = HF.null_check(record.referencedPost_pkey.id)
self.referenced_postreaction = HF.null_check(
record.referencedPostreaction_pkey.id)
self.reaction_type = HF.turn_lower(
HF.null_check(lbsn.PostReaction().ReactionType.Name(
record.reaction_type)))
self.reaction_date = HF.null_check_datetime(record.reaction_date)
self.reaction_content = HF.null_check(record.reaction_content)
self.reaction_like_count = HF.null_check(record.reaction_like_count)
self.user_mentions = list(
set([pkey.id for pkey in record.user_mentions_pkey]))
def dict_selector(self, record):
""" Get dictionary by record type name"""
dict_switcher = {
lbsn.Post().DESCRIPTOR.name: self.lbsn_post_dict,
lbsn.Country().DESCRIPTOR.name: self.lbsn_country_dict,
lbsn.City().DESCRIPTOR.name: self.lbsn_city_dict,
lbsn.Place().DESCRIPTOR.name: self.lbsn_place_dict,
lbsn.PostReaction().DESCRIPTOR.name: self.lbsn_post_reaction_dict,
lbsn.User().DESCRIPTOR.name: self.lbsn_user_dict,
lbsn.UserGroup().DESCRIPTOR.name: self.lbsn_user_group_dict,
lbsn.Origin().DESCRIPTOR.name: self.lbsn_origin_dict
}
return dict_switcher.get(record.DESCRIPTOR.name)
def map_postrecord_to_postreactionrecord(self, post_record):
post_reaction_record = lbsn.PostReaction()
post_reaction_record.pkey.CopyFrom(post_record.pkey)
post_reaction_record.user_pkey.CopyFrom(post_record.user_pkey)
post_reaction_record.reaction_latlng = post_record.post_latlng
# better post_create_date, but not available from Twitter
post_reaction_record.reaction_date.CopyFrom(
post_record.post_publish_date)
post_reaction_record.reaction_like_count = post_record.post_like_count
post_reaction_record.reaction_content = post_record.post_body
post_reaction_record.user_mentions_pkey.extend(
[userRefPkey for userRefPkey in post_record.user_mentions_pkey])
return post_reaction_record
def type_sql_mapper(cls):
"""Assigns record types to SQL Insert SQLs"""
type_sql_mapping = {
lbsn.Origin().DESCRIPTOR.name: cls.origin_insertsql,
lbsn.Country().DESCRIPTOR.name: cls.country_insertsql,
lbsn.City().DESCRIPTOR.name: cls.city_insertsql,
lbsn.Place().DESCRIPTOR.name: cls.place_insertsql,
lbsn.User().DESCRIPTOR.name: cls.user_insertsql,
lbsn.UserGroup().DESCRIPTOR.name: cls.usergroup_insertsql,
lbsn.Post().DESCRIPTOR.name: cls.post_insertsql,
lbsn.Event().DESCRIPTOR.name: cls.event_insertsql,
lbsn.PostReaction().DESCRIPTOR.name: cls.postreaction_insertsql,
}
return type_sql_mapping
def map_postrecord_to_postreactionrecord(cls, post_record):
"""Reduces lbsn.Post to lbsn.PostReaction record"""
post_reaction_record = lbsn.PostReaction()
post_reaction_record.pkey.CopyFrom(post_record.pkey)
post_reaction_record.user_pkey.CopyFrom(post_record.user_pkey)
post_reaction_record.reaction_latlng = post_record.post_latlng
# better post_create_date, but not available from Twitter
post_reaction_record.reaction_date.CopyFrom(
post_record.post_publish_date)
post_reaction_record.reaction_like_count = post_record.post_like_count
post_reaction_record.reaction_content = post_record.post_body
post_reaction_record.user_mentions_pkey.extend(
post_record.user_mentions_pkey)
return post_reaction_record
def get_hll_metrics(cls, record) -> hll.HllMetrics:
"""Extracts hll metrics based on record type"""
dict_switcher = {
lbsn.Origin().DESCRIPTOR.name: cls.get_origin_metrics,
lbsn.Country().DESCRIPTOR.name: cls.get_country_metrics,
lbsn.City().DESCRIPTOR.name: cls.get_city_metrics,
lbsn.Place().DESCRIPTOR.name: cls.get_place_metrics,
lbsn.User().DESCRIPTOR.name: cls.get_user_metrics,
lbsn.UserGroup().DESCRIPTOR.name: cls.get_usergroup_metrics,
lbsn.Post().DESCRIPTOR.name: cls.get_post_metrics,
lbsn.PostReaction().DESCRIPTOR.name: cls.get_postreaction_metrics,
lbsn.Relationship().DESCRIPTOR.name: cls.get_relationship_metrics
}
extract_function = dict_switcher.get(record.DESCRIPTOR.name)
record_hll_metrics = extract_function(record)
return record_hll_metrics
def func_prepare_selector(self, record):
"""Select correct prepare function according to record type"""
dict_switcher = {
lbsn.Origin().DESCRIPTOR.name: self.prepare_lbsn_origin,
lbsn.Country().DESCRIPTOR.name: self.prepare_lbsn_country,
lbsn.City().DESCRIPTOR.name: self.prepare_lbsn_city,
lbsn.Place().DESCRIPTOR.name: self.prepare_lbsn_place,
lbsn.User().DESCRIPTOR.name: self.prepare_lbsn_user,
lbsn.UserGroup().DESCRIPTOR.name: self.prepare_lbsn_usergroup,
lbsn.Post().DESCRIPTOR.name: self.prepare_lbsn_post,
lbsn.Event().DESCRIPTOR.name: self.prepare_lbsn_event,
lbsn.PostReaction().DESCRIPTOR.name:
self.prepare_lbsn_postreaction,
lbsn.Relationship().DESCRIPTOR.name: self.prepare_lbsn_relation
}
prepare_function = dict_switcher.get(record.DESCRIPTOR.name)
return prepare_function(record)
def get_func_record(cls,
record: Dict[str, Any],
input_type: Optional[str] = None):
"""Returns mapping function for input_type"""
FUNC_MAP = {
lbsn.Origin().DESCRIPTOR.name: cls.extract_origin,
lbsn.Country().DESCRIPTOR.name: cls.extract_country,
lbsn.City().DESCRIPTOR.name: cls.extract_city,
lbsn.Place().DESCRIPTOR.name: cls.extract_place,
lbsn.UserGroup().DESCRIPTOR.name: cls.extract_usergroup,
lbsn.User().DESCRIPTOR.name: cls.extract_user,
lbsn.Post().DESCRIPTOR.name: cls.extract_post,
lbsn.PostReaction().DESCRIPTOR.name: cls.extract_postreaction,
lbsn.Event().DESCRIPTOR.name: cls.extract_event,
}
func_map = FUNC_MAP.get(input_type)
# create origin always the same
origin = lbsn.Origin()
origin.origin_id = record.get('origin_id')
return func_map(record, origin)
def __init__(self):
self.lbsn_origin_dict = dict()
self.lbsn_country_dict = dict()
self.lbsn_city_dict = dict()
self.lbsn_place_dict = dict()
self.lbsn_user_group_dict = dict()
self.lbsn_user_dict = dict()
self.lbsn_post_dict = dict()
self.lbsn_post_reaction_dict = dict()
self.lbsn_relationship_dict = dict()
self.key_hashes = {
lbsn.Origin.DESCRIPTOR.name: set(),
lbsn.Post.DESCRIPTOR.name: set(),
lbsn.Country.DESCRIPTOR.name: set(),
lbsn.City.DESCRIPTOR.name: set(),
lbsn.Place.DESCRIPTOR.name: set(),
lbsn.UserGroup.DESCRIPTOR.name: set(),
lbsn.User.DESCRIPTOR.name: set(),
lbsn.PostReaction.DESCRIPTOR.name: set(),
lbsn.Relationship.DESCRIPTOR.name: set()
}
self.count_glob = 0 # total number of records added
self.count_glob_total = 0
self.count_dup_merge = 0 # number of duplicate records merged
self.count_dup_merge_total = 0
# returns all recordsDicts in correct order,
# with names as references (tuple)
self.all_dicts = [
(self.lbsn_origin_dict, lbsn.Origin().DESCRIPTOR.name),
(self.lbsn_country_dict, lbsn.Country().DESCRIPTOR.name),
(self.lbsn_city_dict, lbsn.City().DESCRIPTOR.name),
(self.lbsn_place_dict, lbsn.Place().DESCRIPTOR.name),
(self.lbsn_user_group_dict, lbsn.UserGroup().DESCRIPTOR.name),
(self.lbsn_user_dict, lbsn.User().DESCRIPTOR.name),
(self.lbsn_post_dict, lbsn.Post().DESCRIPTOR.name),
(self.lbsn_post_reaction_dict,
lbsn.PostReaction().DESCRIPTOR.name),
(self.lbsn_relationship_dict, lbsn.Relationship().DESCRIPTOR.name)
]
"""
import enum
from typing import Union, Optional, List, Tuple
from lbsnstructure import lbsnstructure_pb2 as lbsn
"""Schema convention from lbsn db spec"""
LBSN_SCHEMA = [
(lbsn.Origin().DESCRIPTOR.name, "social", "origin", "origin_id"),
(lbsn.Country().DESCRIPTOR.name, "spatial", "country", "country_guid"),
(lbsn.City().DESCRIPTOR.name, "spatial", "city", "city_guid"),
(lbsn.Place().DESCRIPTOR.name, "spatial", "place", "place_guid"),
(lbsn.UserGroup().DESCRIPTOR.name, "social", "user_groups", "usergroup_guid"),
(lbsn.User().DESCRIPTOR.name, "social", "user", "user_guid"),
(lbsn.Post().DESCRIPTOR.name, "topical", "post", "post_guid"),
(lbsn.PostReaction().DESCRIPTOR.name,
"topical", "post_reaction", "reaction_guid"),
]
def optional_schema_override(
LBSN_SCHEMA: List[Tuple[str, str, str, str]],
schema_table_overrides: List[Tuple[str, str]]) -> List[Tuple[str, str, str, str]]:
"""Override schema and table name for selected lbsn objects."""
LBSN_SCHEMA_OVERRIDE = []
for lbsn_type, schema_name, table_name, key_col in LBSN_SCHEMA:
for schema_table_override in schema_table_overrides:
lbsn_object_ref, schema_table_override = schema_table_override
try:
schema_override, table_override = schema_table_override.split(
".")