def _sync(self):
"""Sync conversation state and events that could have been missed."""
logger.info("Syncing events since {}".format(self._sync_timestamp))
try:
res = yield from self._client.sync_all_new_events(
hangouts_pb2.SyncAllNewEventsRequest(
request_header=self._client.get_request_header(),
last_sync_timestamp=parsers.to_timestamp(self._sync_timestamp),
max_response_size_bytes=1048576, # 1 MB
)
)
except exceptions.NetworkError as e:
logger.warning("Failed to sync events, some events may be lost: {}".format(e))
else:
for conv_state in res.conversation_state:
conv_id = conv_state.conversation_id.id
conv = self._conv_dict.get(conv_id, None)
if conv is not None:
conv.update_conversation(conv_state.conversation)
for event_ in conv_state.event:
timestamp = parsers.from_timestamp(event_.timestamp)
if timestamp > self._sync_timestamp:
# This updates the sync_timestamp for us, as well
# as triggering events.
yield from self._on_event(event_)
else:
self.add_conversation(conv_state.conversation, conv_state.event)
def _sync(self, initial_data=None):
"""Sync conversation state and events that could have been missed."""
logger.info('Syncing events since {}'.format(self._sync_timestamp))
try:
res = yield from self._client.syncallnewevents(
self._sync_timestamp
)
except exceptions.NetworkError as e:
logger.warning('Failed to sync events, some events may be lost: {}'
.format(e))
else:
for conv_state in res.conversation_state:
conv_id = conv_state.conversation_id.id_
conv = self._conv_dict.get(conv_id, None)
if conv is not None:
conv.update_conversation(conv_state.conversation)
for event_ in conv_state.event:
timestamp = parsers.from_timestamp(event_.timestamp)
if timestamp > self._sync_timestamp:
# This updates the sync_timestamp for us, as well
# as triggering events.
yield from self._on_client_event(event_)
else:
self.add_conversation(conv_state.conversation,
conv_state.event)
def _sync(self):
"""Sync conversation state and events that could have been missed."""
logger.info('Syncing events since {}'.format(self._sync_timestamp))
try:
res = yield from self._client.sync_all_new_events(
hangouts_pb2.SyncAllNewEventsRequest(
request_header=self._client.get_request_header(),
last_sync_timestamp=parsers.to_timestamp(
self._sync_timestamp),
max_response_size_bytes=1048576, # 1 MB
))
except exceptions.NetworkError as e:
logger.warning(
'Failed to sync events, some events may be lost: {}'.format(e))
else:
for conv_state in res.conversation_state:
conv_id = conv_state.conversation_id.id
conv = self._conv_dict.get(conv_id, None)
if conv is not None:
conv.update_conversation(conv_state.conversation)
for event_ in conv_state.event:
timestamp = parsers.from_timestamp(event_.timestamp)
if timestamp > self._sync_timestamp:
# This updates the sync_timestamp for us, as well
# as triggering events.
yield from self._on_event(event_)
else:
self._add_conversation(conv_state.conversation,
conv_state.event)
def _sync(self, initial_data=None):
"""Sync conversation state and events that could have been missed."""
logger.info('Syncing events since {}'.format(self._sync_timestamp))
try:
res = yield from self._client.syncallnewevents(
self._sync_timestamp
)
except exceptions.NetworkError as e:
logger.warning('Failed to sync events, some events may be lost: {}'
.format(e))
else:
for conv_state in res.conversation_state:
conv_id = conv_state.conversation_id.id_
conv = self._conv_dict.get(conv_id, None)
if conv is not None:
conv.update_conversation(conv_state.conversation)
for event_ in conv_state.event:
timestamp = parsers.from_timestamp(event_.timestamp)
if timestamp > self._sync_timestamp:
# This updates the sync_timestamp for us, as well
# as triggering events.
yield from self._on_client_event(event_)
else:
self.add_conversation(conv_state.conversation,
conv_state.event)
def last_modified(self):
"""When conversation was last modified (:class:`datetime.datetime`)."""
timestamp = self._conversation.self_conversation_state.sort_timestamp
# timestamp can be None for some reason when there is an ongoing video
# hangout
if timestamp is None:
timestamp = 0
return parsers.from_timestamp(timestamp)
def last_modified(self):
"""datetime timestamp of when the conversation was last modified."""
timestamp = self._conversation.self_conversation_state.sort_timestamp
# timestamp can be None for some reason when there is an ongoing video
# hangout
if timestamp is None:
timestamp = 0
return parsers.from_timestamp(timestamp)
def build_user_conversation_list(client):
"""Return UserList from initial contact data and an additional request.
The initial data contains the user's contacts, but there may be conversions
containing users that are not in the contacts. This function takes care of
requesting data for those users and constructing the UserList.
"""
# Retrieve recent conversations so we can preemptively look up their
# participants.
sync_recent_conversations_response = (
yield from client.syncrecentconversations()
)
conv_states = sync_recent_conversations_response.conversation_state
sync_timestamp = parsers.from_timestamp(
# syncrecentconversations seems to return a sync_timestamp 4 minutes
# before the present. To prevent syncallnewevents later breaking
# requesting events older than what we already have, use
# current_server_time instead.
sync_recent_conversations_response.response_header.current_server_time
)
# Retrieve entities participating in all conversations.
required_user_ids = set()
for conv_state in conv_states:
required_user_ids |= {
user.UserID(chat_id=part.id.chat_id, gaia_id=part.id.gaia_id)
for part in conv_state.conversation.participant_data
}
required_entities = []
if required_user_ids:
logger.debug('Need to request additional users: {}'
.format(required_user_ids))
try:
response = yield from client.getentitybyid(
[user_id.gaia_id for user_id in required_user_ids]
)
required_entities = list(response.entity)
except exceptions.NetworkError as e:
logger.warning('Failed to request missing users: {}'.format(e))
# Build list of conversation participants.
conv_part_list = []
for conv_state in conv_states:
conv_part_list.extend(conv_state.conversation.participant_data)
# Retrieve self entity.
get_self_info_response = yield from client.getselfinfo()
self_entity = get_self_info_response.self_entity
user_list = user.UserList(client, self_entity, required_entities,
conv_part_list)
conversation_list = ConversationList(client, conv_states, user_list,
sync_timestamp)
return (user_list, conversation_list)
def _on_client_event(self, event_):
"""Receive a ClientEvent and fan out to Conversations."""
self._sync_timestamp = parsers.from_timestamp(event_.timestamp)
try:
conv = self._conv_dict[event_.conversation_id.id_]
except KeyError:
logger.warning('Received ClientEvent for unknown conversation {}'
.format(event_.conversation_id.id_))
else:
conv_event = conv.add_event(event_)
yield from self.on_event.fire(conv_event)
yield from conv.on_event.fire(conv_event)
def _on_client_event(self, event_):
"""Receive a ClientEvent and fan out to Conversations."""
self._sync_timestamp = parsers.from_timestamp(event_.timestamp)
try:
conv = self._conv_dict[event_.conversation_id.id_]
except KeyError:
logger.warning('Received ClientEvent for unknown conversation {}'
.format(event_.conversation_id.id_))
else:
conv_event = conv.add_event(event_)
yield from self.on_event.fire(conv_event)
yield from conv.on_event.fire(conv_event)
def build_user_conversation_list(client):
"""Return UserList from initial contact data and an additional request.
The initial data contains the user's contacts, but there may be conversions
containing users that are not in the contacts. This function takes care of
requesting data for those users and constructing the UserList.
"""
# Retrieve recent conversations so we can preemptively look up their
# participants.
sync_recent_conversations_response = (yield from
client.syncrecentconversations())
conv_states = sync_recent_conversations_response.conversation_state
sync_timestamp = parsers.from_timestamp(
# syncrecentconversations seems to return a sync_timestamp 4 minutes
# before the present. To prevent syncallnewevents later breaking
# requesting events older than what we already have, use
# current_server_time instead.
sync_recent_conversations_response.response_header.current_server_time)
# Retrieve entities participating in all conversations.
required_user_ids = set()
for conv_state in conv_states:
required_user_ids |= {
user.UserID(chat_id=part.id.chat_id, gaia_id=part.id.gaia_id)
for part in conv_state.conversation.participant_data
}
required_entities = []
if required_user_ids:
logger.debug(
'Need to request additional users: {}'.format(required_user_ids))
try:
response = yield from client.getentitybyid(
[user_id.gaia_id for user_id in required_user_ids])
required_entities = list(response.entity)
except exceptions.NetworkError as e:
logger.warning('Failed to request missing users: {}'.format(e))
# Build list of conversation participants.
conv_part_list = []
for conv_state in conv_states:
conv_part_list.extend(conv_state.conversation.participant_data)
# Retrieve self entity.
get_self_info_response = yield from client.getselfinfo()
self_entity = get_self_info_response.self_entity
user_list = user.UserList(client, self_entity, required_entities,
conv_part_list)
conversation_list = ConversationList(client, conv_states, user_list,
sync_timestamp)
return (user_list, conversation_list)
def _on_event(self, event_):
"""Receive a hangouts_pb2.Event and fan out to Conversations."""
self._sync_timestamp = parsers.from_timestamp(event_.timestamp)
try:
conv = self._conv_dict[event_.conversation_id.id]
except KeyError:
logger.warning("Received Event for unknown conversation {}".format(event_.conversation_id.id))
else:
conv_event = conv.add_event(event_)
# conv_event may be None if the event was a duplicate.
if conv_event is not None:
yield from self.on_event.fire(conv_event)
yield from conv.on_event.fire(conv_event)
def _on_event(self, event_):
"""Receive a hangouts_pb2.Event and fan out to Conversations."""
self._sync_timestamp = parsers.from_timestamp(event_.timestamp)
try:
conv = self._conv_dict[event_.conversation_id.id]
except KeyError:
logger.warning('Received Event for unknown conversation {}'.format(
event_.conversation_id.id))
else:
conv_event = conv.add_event(event_)
# conv_event may be None if the event was a duplicate.
if conv_event is not None:
yield from self.on_event.fire(conv_event)
yield from conv.on_event.fire(conv_event)
async def _sync_all_conversations(client):
"""Sync all conversations by making paginated requests.
Conversations are ordered by ascending sort timestamp.
Args:
client (Client): Connected client.
Raises:
NetworkError: If the requests fail.
Returns:
tuple of list of ``ConversationState`` messages and sync timestamp
"""
conv_states = []
sync_timestamp = None
request = hangouts_pb2.SyncRecentConversationsRequest(
request_header=client.get_request_header(),
max_conversations=CONVERSATIONS_PER_REQUEST,
max_events_per_conversation=1,
sync_filter=[
hangouts_pb2.SYNC_FILTER_INBOX,
hangouts_pb2.SYNC_FILTER_ARCHIVED,
]
)
for _ in range(MAX_CONVERSATION_PAGES):
logger.info(
'Requesting conversations page %s', request.last_event_timestamp
)
response = await client.sync_recent_conversations(request)
conv_states = list(response.conversation_state) + conv_states
sync_timestamp = parsers.from_timestamp(
# SyncRecentConversations seems to return a sync_timestamp 4
# minutes before the present. To prevent SyncAllNewEvents later
# breaking requesting events older than what we already have, use
# current_server_time instead.
response.response_header.current_server_time
)
if response.continuation_end_timestamp == 0:
logger.info('Reached final conversations page')
break
else:
request.last_event_timestamp = response.continuation_end_timestamp
else:
logger.warning('Exceeded maximum number of conversation pages')
logger.info('Synced %s total conversations', len(conv_states))
return conv_states, sync_timestamp
async def _sync_all_conversations(client):
"""Sync all conversations by making paginated requests.
Conversations are ordered by ascending sort timestamp.
Args:
client (Client): Connected client.
Raises:
NetworkError: If the requests fail.
Returns:
tuple of list of ``ConversationState`` messages and sync timestamp
"""
conv_states = []
sync_timestamp = None
request = hangouts_pb2.SyncRecentConversationsRequest(
request_header=client.get_request_header(),
max_conversations=CONVERSATIONS_PER_REQUEST,
max_events_per_conversation=1,
sync_filter=[
hangouts_pb2.SYNC_FILTER_INBOX,
hangouts_pb2.SYNC_FILTER_ARCHIVED,
]
)
for _ in range(MAX_CONVERSATION_PAGES):
logger.info(
'Requesting conversations page %s', request.last_event_timestamp
)
response = await client.sync_recent_conversations(request)
conv_states = list(response.conversation_state) + conv_states
sync_timestamp = parsers.from_timestamp(
# SyncRecentConversations seems to return a sync_timestamp 4
# minutes before the present. To prevent SyncAllNewEvents later
# breaking requesting events older than what we already have, use
# current_server_time instead.
response.response_header.current_server_time
)
if response.continuation_end_timestamp == 0:
logger.info('Reached final conversations page')
break
else:
request.last_event_timestamp = response.continuation_end_timestamp
else:
logger.warning('Exceeded maximum number of conversation pages')
logger.info('Synced %s total conversations', len(conv_states))
return conv_states, sync_timestamp
def _on_event(self, event_):
"""Receive a hangouts_pb2.Event and fan out to Conversations.
Args:
event_: hangouts_pb2.Event instance
"""
conv_id = event_.conversation_id.id
try:
conv = yield from self._get_or_fetch_conversation(conv_id)
except exceptions.NetworkError:
logger.warning(
'Failed to fetch conversation for event notification: %s',
conv_id)
else:
self._sync_timestamp = parsers.from_timestamp(event_.timestamp)
conv_event = conv.add_event(event_)
# conv_event may be None if the event was a duplicate.
if conv_event is not None:
yield from self.on_event.fire(conv_event)
yield from conv.on_event.fire(conv_event)
async def _on_event(self, event_):
"""Receive a hangouts_pb2.Event and fan out to Conversations.
Args:
event_: hangouts_pb2.Event instance
"""
conv_id = event_.conversation_id.id
try:
conv = await self._get_or_fetch_conversation(conv_id)
except exceptions.NetworkError:
logger.warning(
'Failed to fetch conversation for event notification: %s',
conv_id
)
else:
self._sync_timestamp = parsers.from_timestamp(event_.timestamp)
conv_event = conv.add_event(event_)
# conv_event may be None if the event was a duplicate.
if conv_event is not None:
await self.on_event.fire(conv_event)
await conv.on_event.fire(conv_event)
def _update(self, data) -> None:
self.reachable = getattr(data, 'reachable', None)
self.available = getattr(data, 'available', None)
device_status = getattr(data, 'device_status', None)
self.mobile = getattr(device_status, 'mobile', None)
self.desktop = getattr(device_status, 'desktop', None)
self.tablet = getattr(device_status, 'tablet', None)
mood_message = getattr(data, 'mood_message', None)
mood_content = getattr(mood_message, 'mood_content', None)
segment = getattr(mood_content, 'segment', None)
mood_segment = [ChatMessageSegment.deserialize(segment) for segment in segment]
self.mood_message = [segment.text for segment in mood_segment]
last_seen = getattr(data, 'last_seen', None)
self.last_seen = parsers.from_timestamp(getattr(last_seen, 'last_seen_timestamp_usec', None))
self.since_last_seen = getattr(last_seen, 'usec_since_last_seen', None) // 1000000
def update_conversation(self, conversation):
"""Update the internal state of the conversation.
This method is used by :class:`.ConversationList` to maintain this
instance.
Args:
conversation: ``Conversation`` message.
"""
# StateUpdate.conversation is actually a delta; fields that aren't
# specified are assumed to be unchanged. Until this class is
# refactored, hide this by saving and restoring previous values where
# necessary.
new_state = conversation.self_conversation_state
old_state = self._conversation.self_conversation_state
self._conversation = conversation
# delivery_medium_option
if not new_state.delivery_medium_option:
new_state.delivery_medium_option.extend(
old_state.delivery_medium_option
)
# latest_read_timestamp
old_timestamp = old_state.self_read_state.latest_read_timestamp
new_timestamp = new_state.self_read_state.latest_read_timestamp
if new_timestamp == 0:
new_state.self_read_state.latest_read_timestamp = old_timestamp
# user_read_state(s)
for new_entry in conversation.read_state:
tstamp = parsers.from_timestamp(new_entry.latest_read_timestamp)
if tstamp == 0:
continue
uid = parsers.from_participantid(new_entry.participant_id)
if uid not in self._watermarks or self._watermarks[uid] < tstamp:
self._watermarks[uid] = tstamp
def update_conversation(self, conversation):
"""Update the internal state of the conversation.
This method is used by :class:`.ConversationList` to maintain this
instance.
Args:
conversation: ``Conversation`` message.
"""
# StateUpdate.conversation is actually a delta; fields that aren't
# specified are assumed to be unchanged. Until this class is
# refactored, hide this by saving and restoring previous values where
# necessary.
new_state = conversation.self_conversation_state
old_state = self._conversation.self_conversation_state
self._conversation = conversation
# delivery_medium_option
if not new_state.delivery_medium_option:
new_state.delivery_medium_option.extend(
old_state.delivery_medium_option
)
# latest_read_timestamp
old_timestamp = old_state.self_read_state.latest_read_timestamp
new_timestamp = new_state.self_read_state.latest_read_timestamp
if new_timestamp == 0:
new_state.self_read_state.latest_read_timestamp = old_timestamp
# user_read_state(s)
for new_entry in conversation.read_state:
tstamp = parsers.from_timestamp(new_entry.latest_read_timestamp)
if tstamp == 0:
continue
uid = parsers.from_participantid(new_entry.participant_id)
if uid not in self._watermarks or self._watermarks[uid] < tstamp:
self._watermarks[uid] = tstamp
def _sync_chat_messages(self):
"""Sync chat messages since self._sync_timestamp."""
logger.info('Syncing messages since {}'.format(self._sync_timestamp))
res = yield self.syncallnewevents(self._sync_timestamp)
# Parse chat message from response and fire on_message event for each
# new chat message.
conversation_state = res[3]
for conversation in conversation_state:
events = conversation[2]
for msg in events:
try:
chat_message = parsers.parse_chat_message([msg])
except exceptions.ParseError as e:
logger.warning('Failed to parse message: {}'.format(e))
except exceptions.ParseNotImplementedError as e:
logger.info('Failed to parse message: {}'.format(e))
else:
# Workaround for syncallnewevents timestamp being
# inclusive:
if chat_message.timestamp > self._sync_timestamp:
self.on_message.fire(chat_message)
self._sync_timestamp = parsers.from_timestamp(int(res[1][4]))
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
try:
res = yield from http_utils.fetch('get',
CHAT_INIT_URL,
cookies=self._cookies,
params=CHAT_INIT_PARAMS,
connector=self._connector)
except exceptions.NetworkError as e:
raise exceptions.HangupsError(
'Initialize chat request failed: {}'.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
logger.debug(
'Failed to parse initialize chat object: {}\n{}'.format(
e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
_sync_timestamp = parsers.from_timestamp(
data_dict['ds:21'][0][1][4])
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
data_dict['ds:20'][0]).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
data_dict['ds:19'][0][3])
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
data_dict['ds:21'][0])
except ValueError as e:
logger.warning(
'Failed to parse initial client entities: {}'.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity, entities.
group3.entity, entities.group4.entity, entities.group5.entity))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def timestamp(self):
"""A timestamp of when the event occurred."""
return parsers.from_timestamp(self._event.timestamp)
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
# We first need to fetch the 'pvt' token, which is required for the
# initialization request (otherwise it will return 400).
try:
res = yield from http_utils.fetch(
'get', PVT_TOKEN_URL, cookies=self._cookies,
connector=self._connector
)
CHAT_INIT_PARAMS['pvt'] = javascript.loads(res.body.decode())[1]
logger.info('Found PVT token: {}'.format(CHAT_INIT_PARAMS['pvt']))
except (exceptions.NetworkError, ValueError) as e:
raise exceptions.HangupsError('Failed to fetch PVT token: {}'
.format(e))
# Now make the actual initialization request:
try:
res = yield from http_utils.fetch(
'get', CHAT_INIT_URL, cookies=self._cookies,
params=CHAT_INIT_PARAMS, connector=self._connector
)
except exceptions.NetworkError as e:
raise exceptions.HangupsError('Initialize chat request failed: {}'
.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
logger.debug("Attempting to load javascript: {}..."
.format(repr(data[:100])))
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
try:
data = data.replace("data:function(){return", "data:")
data = data.replace("}}", "}")
data = javascript.loads(data)
data_dict[data['key']] = data['data']
except ValueError as e:
raise
# logger.debug('Failed to parse initialize chat object: {}\n{}'
# .format(e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._email = data_dict['ds:34'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
_sync_timestamp = parsers.from_timestamp(
# cgserp?
# data_dict['ds:21'][0][1][4]
# data_dict['ds:35'][0][1][4]
data_dict['ds:21'][0][1][4]
)
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
# cgsirp?
# data_dict['ds:20'][0]
# data_dict['ds:35'][0]
data_dict['ds:20'][0]
).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
# csrcrp?
# data_dict['ds:19'][0][3]
# data_dict['ds:36'][0][3]
data_dict['ds:19'][0][3]
)
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
# cgserp?
# data_dict['ds:21'][0]
# data_dict['ds:37'][0]
data_dict['ds:21'][0]
)
except ValueError as e:
logger.warning('Failed to parse initial client entities: {}'
.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity,
entities.group3.entity, entities.group4.entity,
entities.group5.entity
))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def build_user_conversation_list(client):
"""Build :class:`~UserList` and :class:`~ConversationList`.
This method requests data necessary to build the list of conversations and
users. Users that are not in the contact list but are participating in a
conversation will also be retrieved.
Args:
client (Client): Connected client.
Returns:
(:class:`~UserList`, :class:`~ConversationList`):
Tuple of built objects.
"""
# Retrieve conversations in groups of CONVERSATIONS_PER_REQUEST.
conv_states = []
sync_timestamp, next_timestamp = None, None
last_synced = CONVERSATIONS_PER_REQUEST
while last_synced == CONVERSATIONS_PER_REQUEST:
response = (
yield from client.sync_recent_conversations(
hangouts_pb2.SyncRecentConversationsRequest(
request_header=client.get_request_header(),
last_event_timestamp=next_timestamp,
max_conversations=CONVERSATIONS_PER_REQUEST,
max_events_per_conversation=1,
sync_filter=[
hangouts_pb2.SYNC_FILTER_INBOX,
hangouts_pb2.SYNC_FILTER_ARCHIVED,
]
)
)
)
# Add these conversations to the list of states
response_conv_states = response.conversation_state
min_timestamp = float('inf')
for conv_state in response_conv_states:
conv_event = conv_state.event[0]
if conv_event.timestamp < min_timestamp:
min_timestamp = conv_event.timestamp
conv_states.append(conv_state)
# Update the number of conversations synced and sync timestamp
last_synced = len(response_conv_states)
sync_timestamp = parsers.from_timestamp(
# SyncRecentConversations seems to return a sync_timestamp 4
# minutes before the present. To prevent SyncAllNewEvents later
# breaking requesting events older than what we already have, use
# current_server_time instead.
response.response_header.current_server_time
)
logger.debug('Added {} conversations'.format(last_synced))
if math.isfinite(min_timestamp):
# Start syncing conversations just before this one
next_timestamp = min_timestamp - 1
else:
# No minimum timestamp; abort.
next_timestamp = 0
break
logger.info('Synced {} total conversations'.format(len(conv_states)))
# Retrieve entities participating in all conversations.
required_user_ids = set()
for conv_state in conv_states:
required_user_ids |= {
user.UserID(chat_id=part.id.chat_id, gaia_id=part.id.gaia_id)
for part in conv_state.conversation.participant_data
}
required_entities = []
if required_user_ids:
logger.debug('Need to request additional users: {}'
.format(required_user_ids))
try:
response = yield from client.get_entity_by_id(
hangouts_pb2.GetEntityByIdRequest(
request_header=client.get_request_header(),
batch_lookup_spec=[
hangouts_pb2.EntityLookupSpec(
gaia_id=user_id.gaia_id,
create_offnetwork_gaia=True,
)
for user_id in required_user_ids
],
)
)
for entity_result in response.entity_result:
required_entities.extend(entity_result.entity)
except exceptions.NetworkError as e:
logger.warning('Failed to request missing users: {}'.format(e))
# Build list of conversation participants.
conv_part_list = []
for conv_state in conv_states:
conv_part_list.extend(conv_state.conversation.participant_data)
# Retrieve self entity.
get_self_info_response = yield from client.get_self_info(
hangouts_pb2.GetSelfInfoRequest(
request_header=client.get_request_header(),
)
)
self_entity = get_self_info_response.self_entity
user_list = user.UserList(client, self_entity, required_entities,
conv_part_list)
conversation_list = ConversationList(client, conv_states,
user_list, sync_timestamp)
return (user_list, conversation_list)
def build_user_conversation_list(client):
"""Return UserList from initial contact data and an additional request.
The initial data contains the user's contacts, but there may be conversions
containing users that are not in the contacts. This function takes care of
requesting data for those users and constructing the UserList.
"""
# Retrieve recent conversations so we can preemptively look up their
# participants.
sync_recent_conversations_response = (
yield from client.sync_recent_conversations(
hangouts_pb2.SyncRecentConversationsRequest(
request_header=client.get_request_header(),
max_conversations=100,
max_events_per_conversation=1,
sync_filter=[hangouts_pb2.SYNC_FILTER_INBOX],
)))
conv_states = sync_recent_conversations_response.conversation_state
sync_timestamp = parsers.from_timestamp(
# syncrecentconversations seems to return a sync_timestamp 4 minutes
# before the present. To prevent syncallnewevents later breaking
# requesting events older than what we already have, use
# current_server_time instead.
sync_recent_conversations_response.response_header.current_server_time)
# Retrieve entities participating in all conversations.
required_user_ids = set()
for conv_state in conv_states:
required_user_ids |= {
user.UserID(chat_id=part.id.chat_id, gaia_id=part.id.gaia_id)
for part in conv_state.conversation.participant_data
}
required_entities = []
if required_user_ids:
logger.debug(
'Need to request additional users: {}'.format(required_user_ids))
try:
response = yield from client.get_entity_by_id(
hangouts_pb2.GetEntityByIdRequest(
request_header=client.get_request_header(),
batch_lookup_spec=[
hangouts_pb2.EntityLookupSpec(
gaia_id=user_id.gaia_id,
create_offnetwork_gaia=True,
) for user_id in required_user_ids
],
))
for entity_result in response.entity_result:
required_entities.extend(entity_result.entity)
except exceptions.NetworkError as e:
logger.warning('Failed to request missing users: {}'.format(e))
# Build list of conversation participants.
conv_part_list = []
for conv_state in conv_states:
conv_part_list.extend(conv_state.conversation.participant_data)
# Retrieve self entity.
get_self_info_response = yield from client.get_self_info(
hangouts_pb2.GetSelfInfoRequest(
request_header=client.get_request_header(), ))
self_entity = get_self_info_response.self_entity
user_list = user.UserList(client, self_entity, required_entities,
conv_part_list)
conversation_list = ConversationList(client, conv_states, user_list,
sync_timestamp)
return (user_list, conversation_list)
def latest_read_timestamp(self):
"""Timestamp of latest read event (:class:`datetime.datetime`)."""
timestamp = (self._conversation.self_conversation_state.
self_read_state.latest_read_timestamp)
return parsers.from_timestamp(timestamp)
def latest_read_timestamp(self):
"""datetime timestamp of the last read ConversationEvent."""
timestamp = self._conversation.self_conversation_state.self_read_state.latest_read_timestamp
return parsers.from_timestamp(timestamp)
def build_user_conversation_list(client):
"""Return UserList from initial contact data and an additional request.
The initial data contains the user's contacts, but there may be conversions
containing users that are not in the contacts. This function takes care of
requesting data for those users and constructing the UserList.
"""
# Retrieve recent conversations so we can preemptively look up their
# participants.
sync_recent_conversations_response = (
yield from client.sync_recent_conversations(
hangouts_pb2.SyncRecentConversationsRequest(
request_header=client.get_request_header(),
max_conversations=100,
max_events_per_conversation=1,
sync_filter=[hangouts_pb2.SYNC_FILTER_INBOX],
)
)
)
conv_states = sync_recent_conversations_response.conversation_state
sync_timestamp = parsers.from_timestamp(
# syncrecentconversations seems to return a sync_timestamp 4 minutes
# before the present. To prevent syncallnewevents later breaking
# requesting events older than what we already have, use
# current_server_time instead.
sync_recent_conversations_response.response_header.current_server_time
)
# Retrieve entities participating in all conversations.
required_user_ids = set()
for conv_state in conv_states:
required_user_ids |= {
user.UserID(chat_id=part.id.chat_id, gaia_id=part.id.gaia_id)
for part in conv_state.conversation.participant_data
}
required_entities = []
if required_user_ids:
logger.debug('Need to request additional users: {}'
.format(required_user_ids))
try:
response = yield from client.get_entity_by_id(
hangouts_pb2.GetEntityByIdRequest(
request_header=client.get_request_header(),
batch_lookup_spec=[
hangouts_pb2.EntityLookupSpec(
gaia_id=user_id.gaia_id,
create_offnetwork_gaia=True,
)
for user_id in required_user_ids
],
)
)
for entity_result in response.entity_result:
required_entities.extend(entity_result.entity)
except exceptions.NetworkError as e:
logger.warning('Failed to request missing users: {}'.format(e))
# Build list of conversation participants.
conv_part_list = []
for conv_state in conv_states:
conv_part_list.extend(conv_state.conversation.participant_data)
# Retrieve self entity.
get_self_info_response = yield from client.get_self_info(
hangouts_pb2.GetSelfInfoRequest(
request_header=client.get_request_header(),
)
)
self_entity = get_self_info_response.self_entity
user_list = user.UserList(client, self_entity, required_entities,
conv_part_list)
conversation_list = ConversationList(client, conv_states, user_list,
sync_timestamp)
return (user_list, conversation_list)
def last_modified(self):
"""datetime timestamp of when the conversation was last modified."""
return parsers.from_timestamp(
self._conversation.self_conversation_state.sort_timestamp
)
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
# We first need to fetch the 'pvt' token, which is required for the
# initialization request (otherwise it will return 400).
try:
res = yield from http_utils.fetch('get',
PVT_TOKEN_URL,
cookies=self._cookies,
connector=self._connector)
CHAT_INIT_PARAMS['pvt'] = javascript.loads(res.body.decode())[1]
logger.info('Found PVT token: {}'.format(CHAT_INIT_PARAMS['pvt']))
except (exceptions.NetworkError, ValueError) as e:
raise exceptions.HangupsError(
'Failed to fetch PVT token: {}'.format(e))
# Now make the actual initialization request:
try:
res = yield from http_utils.fetch('get',
CHAT_INIT_URL,
cookies=self._cookies,
params=CHAT_INIT_PARAMS,
connector=self._connector)
except exceptions.NetworkError as e:
raise exceptions.HangupsError(
'Initialize chat request failed: {}'.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
logger.debug("Attempting to load javascript: {}...".format(
repr(data[:100])))
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
try:
data = data.replace("data:function(){return", "data:")
data = data.replace("}}", "}")
data = javascript.loads(data)
data_dict[data['key']] = data['data']
except ValueError as e:
raise
# logger.debug('Failed to parse initialize chat object: {}\n{}'
# .format(e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._email = data_dict['ds:34'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
_sync_timestamp = parsers.from_timestamp(
# cgserp?
# data_dict['ds:21'][0][1][4]
# data_dict['ds:35'][0][1][4]
data_dict['ds:21'][0][1][4])
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
# cgsirp?
# data_dict['ds:20'][0]
# data_dict['ds:35'][0]
data_dict['ds:20'][0]).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
# csrcrp?
# data_dict['ds:19'][0][3]
# data_dict['ds:36'][0][3]
data_dict['ds:19'][0][3])
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
# cgserp?
# data_dict['ds:21'][0]
# data_dict['ds:37'][0]
data_dict['ds:21'][0])
except ValueError as e:
logger.warning(
'Failed to parse initial client entities: {}'.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity, entities.
group3.entity, entities.group4.entity, entities.group5.entity))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def timestamp(self):
"""When the event occurred (:class:`datetime.datetime`)."""
return parsers.from_timestamp(self._event.timestamp)
def latest_read_timestamp(self):
"""Timestamp of latest read event (:class:`datetime.datetime`)."""
timestamp = (self._conversation.self_conversation_state.
self_read_state.latest_read_timestamp)
return parsers.from_timestamp(timestamp)
def last_modified(self):
"""datetime timestamp of when the conversation was last modified."""
return parsers.from_timestamp(
self._conversation.self_conversation_state.sort_timestamp)
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
try:
res = yield from http_utils.fetch(
'get', CHAT_INIT_URL, cookies=self._cookies,
params=CHAT_INIT_PARAMS, connector=self._connector
)
except exceptions.NetworkError as e:
raise exceptions.HangupsError('Initialize chat request failed: {}'
.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
logger.debug('Failed to parse initialize chat object: {}\n{}'
.format(e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
_sync_timestamp = parsers.from_timestamp(
data_dict['ds:21'][0][1][4]
)
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
data_dict['ds:20'][0]
).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
data_dict['ds:19'][0][3]
)
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
data_dict['ds:21'][0]
)
except ValueError as e:
logger.warning('Failed to parse initial client entities: {}'
.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity,
entities.group3.entity, entities.group4.entity,
entities.group5.entity
))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def _init_talkgadget_1(self):
"""Make first talkgadget request and parse response.
The response body is a HTML document containing a series of script tags
containing JavaScript object. We need to parse the object to get at the
data.
"""
url = 'https://talkgadget.google.com/u/0/talkgadget/_/chat'
params = {
'prop': 'aChromeExtension',
'fid': 'gtn-roster-iframe-id',
'ec': '["ci:ec",true,true,false]',
}
headers = {
# appears to require a browser user agent
'user-agent': (
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
'(KHTML, like Gecko) Chrome/34.0.1847.132 Safari/537.36'
),
}
res = yield http_utils.fetch(
url, cookies=self._cookies, params=params, headers=headers,
connect_timeout=CONNECT_TIMEOUT, request_timeout=REQUEST_TIMEOUT
)
logger.debug('First talkgadget request result:\n{}'.format(res.body))
if res.code != 200:
raise ValueError("First talkgadget request failed with {}: {}"
.format(res.code, res.body))
res = res.body.decode()
# Parse the response by using a regex to find all the JS objects, and
# parsing them.
res = res.replace('\n', '')
regex = re.compile(
r"(?:<script>AF_initDataCallback\((.*?)\);</script>)"
)
data_dict = {}
for data in regex.findall(res):
try:
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
# not everything will be parsable, but we don't care
logger.debug('Failed to parse JavaScript: {}\n{}'
.format(e, data))
# TODO: handle errors here
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
self._sync_timestamp = parsers.from_timestamp(
data_dict['ds:21'][0][1][4]
)
# build dict of conversations and their participants
initial_conversations = {}
self.initial_users = {} # {UserID: User}
# add self to the contacts
self_contact = data_dict['ds:20'][0][2]
self.self_user_id = parsers.UserID(chat_id=self_contact[8][0],
gaia_id=self_contact[8][1])
self.initial_users[self.self_user_id] = parsers.User(
id_=self.self_user_id, full_name=self_contact[9][1],
first_name=self_contact[9][2], is_self=True
)
conversations = data_dict['ds:19'][0][3]
for c in conversations:
id_ = c[1][0][0]
participants = c[1][13]
last_modified = c[1][3][12]
# With every converstion, we get a list of up to 20 of the most
# recent messages, sorted oldest to newest.
messages = []
for raw_message in c[2]:
try:
chat_message = parsers.parse_chat_message([raw_message])
except exceptions.ParseError as e:
logger.warning('Failed to parse message: {}'.format(e))
except exceptions.ParseNotImplementedError as e:
logger.info('Failed to parse message: {}'.format(e))
else:
messages.append(chat_message)
initial_conversations[id_] = {
'participants': [],
'last_modified': last_modified,
'name': c[1][2],
'messages': messages,
}
# Add the participants for this conversation.
for p in participants:
user_id = parsers.UserID(chat_id=p[0][0], gaia_id=p[0][1])
initial_conversations[id_]['participants'].append(
user_id
)
# Add the participant to our list of contacts as a fallback, in
# case they can't be found later by other methods.
# TODO We should note who these users are and try to request
# them.
# p[1] can be a full name, None, or out of range.
try:
display_name = p[1]
except IndexError:
display_name = None
if display_name is None:
display_name = 'Unknown'
self.initial_users[user_id] = parsers.User(
id_=user_id, first_name=display_name.split()[0],
full_name=display_name,
is_self=(user_id == self.self_user_id)
)
# build dict of contacts and their names (doesn't include users not in
# contacts)
contacts_main = data_dict['ds:21'][0]
# contacts_main[2] has some, but the format is slightly different
contacts = (contacts_main[4][2] + contacts_main[5][2] +
contacts_main[6][2] + contacts_main[7][2] +
contacts_main[8][2])
for c in contacts:
user_id = parsers.UserID(chat_id=c[0][8][0], gaia_id=c[0][8][1])
self.initial_users[user_id] = parsers.User(
id_=user_id, full_name=c[0][9][1], first_name=c[0][9][2],
is_self=(user_id == self.self_user_id)
)
# Create a dict of the known conversations.
self.initial_conversations = {conv_id: Conversation(
self, conv_id, [self.initial_users[user_id] for user_id
in conv_info['participants']],
conv_info['last_modified'], conv_info['name'],
conv_info['messages'],
) for conv_id, conv_info in initial_conversations.items()}
def latest_read_timestamp(self):
"""datetime timestamp of the last read ConversationEvent."""
timestamp = (self._conversation.self_conversation_state.\
self_read_state.latest_read_timestamp)
return parsers.from_timestamp(timestamp)
def timestamp(self):
return parsers.from_timestamp(self._event.timestamp)
