def consume(self, event): # type: (Mapping[str, Any]) -> None user_profile = get_user_profile_by_id(event["user_profile_id"]) client = get_client(event["client"]) log_time = timestamp_to_datetime(event["time"]) query = event["query"] do_update_user_activity(user_profile, client, query, log_time)
def consume(self, event): # type: (Mapping[str, Any]) -> None user_profile = get_user_profile_by_id(event["user_profile_id"]) client = get_client(event["client"]) log_time = timestamp_to_datetime(event["time"]) query = event["query"] do_update_user_activity(user_profile, client, query, log_time)
def consume_batch(self, user_activity_events: List[Dict[str, Any]]) -> None: uncommitted_events = {} # type: Dict[Tuple[int, int, str], Tuple[int, float]] # First, we drain the queue of all user_activity events and # deduplicate them for insertion into the database. for event in user_activity_events: user_profile_id = event["user_profile_id"] if event["client"] not in self.client_id_map: client = get_client(event["client"]) self.client_id_map[event["client"]] = client.id client_id = self.client_id_map[event["client"]] key_tuple = (user_profile_id, client_id, event["query"]) if key_tuple not in uncommitted_events: uncommitted_events[key_tuple] = (1, event['time']) else: count, time = uncommitted_events[key_tuple] uncommitted_events[key_tuple] = (count + 1, max(time, event['time'])) # Then we insert the updates into the database. # # TODO: Doing these updates in sequence individually is likely # inefficient; the idealized version would do some sort of # bulk insert_or_update query. for key_tuple in uncommitted_events: (user_profile_id, client_id, query) = key_tuple count, time = uncommitted_events[key_tuple] log_time = timestamp_to_datetime(time) do_update_user_activity(user_profile_id, client_id, query, count, log_time)
def consume_batch(self, user_activity_events: List[Dict[str, Any]]) -> None: uncommitted_events: Dict[Tuple[int, int, str], Tuple[int, float]] = {} # First, we drain the queue of all user_activity events and # deduplicate them for insertion into the database. for event in user_activity_events: user_profile_id = event["user_profile_id"] if "client_id" not in event: # This is for compatibility with older events still stuck in the queue, # that used the client name in event["client"] instead of having # event["client_id"] directly. # # TODO/compatibility: We can delete this once it is no # longer possible to directly upgrade from 2.1 to main. if event["client"] not in self.client_id_map: client = get_client(event["client"]) self.client_id_map[event["client"]] = client.id client_id = self.client_id_map[event["client"]] else: client_id = event["client_id"] key_tuple = (user_profile_id, client_id, event["query"]) if key_tuple not in uncommitted_events: uncommitted_events[key_tuple] = (1, event["time"]) else: count, time = uncommitted_events[key_tuple] uncommitted_events[key_tuple] = (count + 1, max(time, event["time"])) # Then we insert the updates into the database. # # TODO: Doing these updates in sequence individually is likely # inefficient; the idealized version would do some sort of # bulk insert_or_update query. for key_tuple in uncommitted_events: (user_profile_id, client_id, query) = key_tuple count, time = uncommitted_events[key_tuple] log_time = timestamp_to_datetime(time) do_update_user_activity(user_profile_id, client_id, query, count, log_time)