def _base_request(self, url, content_type, response_type, data):
"""Send a generic authenticated POST request.
Args:
url (str): URL of request.
content_type (str): Request content type.
response_type (str): The desired response format. Valid options
are: 'json' (JSON), 'protojson' (pblite), and 'proto' (binary
Protocol Buffer). 'proto' requires manually setting an extra
header 'X-Goog-Encode-Response-If-Executable: base64'.
data (str): Request body data.
Returns:
FetchResponse: Response containing HTTP code, cookies, and body.
Raises:
NetworkError: If the request fails.
"""
sapisid_cookie = self._get_cookie('SAPISID')
headers = channel.get_authorization_headers(sapisid_cookie)
headers['content-type'] = content_type
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {cookie: self._get_cookie(cookie)
for cookie in required_cookies}
params = {
# "alternative representation type" (desired response format).
'alt': response_type,
}
res = yield from http_utils.fetch(
'post', url, headers=headers, cookies=cookies, params=params,
data=data, connector=self._connector
)
return res
def _fetch_channel_sid(self):
"""Request a new session ID for the push channel.
Raises hangups.NetworkError.
"""
logger.info('Requesting new session...')
url = 'https://talkgadget.google.com{}bind'.format(self._channel_path)
params = {
'VER': 8,
'clid': self._clid_param,
'ec': self._ec_param,
'RID': 81187,
# Required if we want our client to be called "AChromeExtension":
'prop': self._prop_param,
}
try:
res = yield from http_utils.fetch('post',
url,
cookies=self._cookies,
params=params,
data='count=0',
connector=self._connector)
except exceptions.NetworkError as e:
raise exceptions.HangupsError(
'Failed to request SID: {}'.format(e))
# TODO: Re-write the function we're calling here to use a schema so we
# can easily catch its failure.
self._sid_param, self._email, self._header_client, self._gsessionid_param = (
_parse_sid_response(res.body))
logger.info('New SID: {}'.format(self._sid_param))
logger.info('New email: {}'.format(self._email))
logger.info('New client: {}'.format(self._header_client))
logger.info('New gsessionid: {}'.format(self._gsessionid_param))
def send_maps(self, map_list):
"""Sends a request to the server containing maps (dicts)."""
params = {
'VER': 8, # channel protocol version
'RID': 81188, # request identifier
'ctype': 'hangouts', # client type
}
if self._gsessionid_param is not None:
params['gsessionid'] = self._gsessionid_param
if self._sid_param is not None:
params['SID'] = self._sid_param
data_dict = dict(count=len(map_list), ofs=0)
for map_num, map_ in enumerate(map_list):
for map_key, map_val in map_.items():
data_dict['req{}_{}'.format(map_num, map_key)] = map_val
res = yield from http_utils.fetch(
'post',
CHANNEL_URL_PREFIX.format('channel/bind'),
cookies=self._cookies,
connector=self._connector,
headers=get_authorization_headers(self._cookies['SAPISID']),
params=params,
data=data_dict,
)
return res
def _fetch_channel_sid(self):
"""Request a new session ID for the push channel."""
logger.info('Requesting new session ID...')
url = 'https://talkgadget.google.com{}bind'.format(self._channel_path)
params = {
'VER': 8,
'clid': self._clid,
'ec': self._channel_ec_param,
'RID': 81187, # TODO: "request ID"? should probably increment
# Required if we want our client to be called "AChromeExtension":
'prop': self._channel_prop_param,
}
res = yield http_utils.fetch(
url, method='POST', cookies=self._cookies, params=params,
data='count=0', connect_timeout=CONNECT_TIMEOUT,
request_timeout=REQUEST_TIMEOUT
)
logger.debug('Fetch SID response:\n{}'.format(res.body))
if res.code != 200:
# TODO use better exception
raise ValueError("SID fetch request failed with {}: {}"
.format(res.code, res.raw.read()))
# TODO: handle errors here
self._channel_session_id, self._header_client, self._gsessionid = (
_parse_sid_response(res.body)
)
logger.info('Received new session ID: {}'
.format(self._channel_session_id))
def _request(self, endpoint, body_json):
"""Make chat API request."""
url = 'https://clients6.google.com/chat/v1/{}'.format(endpoint)
headers = {
'authorization': self._get_authorization_header(),
'x-origin': ORIGIN_URL,
'x-goog-authuser': '******',
'content-type': 'application/json+protobuf',
}
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {
cookie: self._get_cookie(cookie)
for cookie in required_cookies
}
params = {
'key': self._api_key,
'alt': 'json', # json or protojson
}
res = yield http_utils.fetch(url,
method='POST',
headers=headers,
cookies=cookies,
params=params,
data=json.dumps(body_json),
request_timeout=REQUEST_TIMEOUT,
connect_timeout=CONNECT_TIMEOUT)
logger.debug('Response to request for {} was {}:\n{}'.format(
endpoint, res.code, res.body))
if res.code != 200:
raise ValueError(
'Request to {} endpoint failed with {}: {}'.format(
endpoint, res.code, res.body.decode()))
return res
def _request(self, endpoint, body_json, use_json=True):
"""Make chat API request.
Raises hangups.NetworkError if the request fails.
"""
url = 'https://clients6.google.com/chat/v1/{}'.format(endpoint)
headers = {
'authorization': self._get_authorization_header(),
'x-origin': ORIGIN_URL,
'x-goog-authuser': '******',
'content-type': 'application/json+protobuf',
}
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {
cookie: self._get_cookie(cookie)
for cookie in required_cookies
}
params = {
'key': self._api_key,
'alt': 'json' if use_json else 'protojson',
}
logger.debug("Fetching '{}' with '{}'".format(url, body_json))
res = yield from http_utils.fetch('post',
url,
headers=headers,
cookies=cookies,
params=params,
data=json.dumps(body_json),
connector=self._connector)
logger.debug('Response to request for {} was {}:\n{}'.format(
endpoint, res.code, res.body))
return res
def _base_request(self, url, content_type, data, use_json=True):
"""Make API request.
Raises hangups.NetworkError if the request fails.
"""
headers = channel.get_authorization_headers(
self._get_cookie('SAPISID'))
headers['content-type'] = content_type
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {
cookie: self._get_cookie(cookie)
for cookie in required_cookies
}
params = {
'key': self._api_key,
'alt': 'json' if use_json else 'protojson',
}
res = yield from http_utils.fetch('post',
url,
headers=headers,
cookies=cookies,
params=params,
data=data,
connector=self._connector)
logger.debug('Response to request for {} was {}:\n{}'.format(
url, res.code, res.body))
return res
def _fetch_channel_sid(self):
"""Request a new session ID for the push channel."""
logger.info('Requesting new session ID...')
url = 'https://talkgadget.google.com{}bind'.format(self._channel_path)
params = {
'VER': 8,
'clid': self._clid,
'ec': self._channel_ec_param,
'RID': 81187, # TODO: "request ID"? should probably increment
# Required if we want our client to be called "AChromeExtension":
'prop': self._channel_prop_param,
}
res = yield http_utils.fetch(url,
method='POST',
cookies=self._cookies,
params=params,
data='count=0',
connect_timeout=CONNECT_TIMEOUT,
request_timeout=REQUEST_TIMEOUT)
logger.debug('Fetch SID response:\n{}'.format(res.body))
if res.code != 200:
# TODO use better exception
raise ValueError("SID fetch request failed with {}: {}".format(
res.code, res.raw.read()))
# TODO: handle errors here
self._channel_session_id, self._header_client, self._gsessionid = (
_parse_sid_response(res.body))
logger.info('Received new session ID: {}'.format(
self._channel_session_id))
def _request(self, endpoint, body_json, use_json=True):
"""Make chat API request.
Raises hangups.NetworkError if the request fails.
"""
url = 'https://clients6.google.com/chat/v1/{}'.format(endpoint)
headers = {
'authorization': self._get_authorization_header(),
'x-origin': ORIGIN_URL,
'x-goog-authuser': '******',
'content-type': 'application/json+protobuf',
}
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {cookie: self._get_cookie(cookie)
for cookie in required_cookies}
params = {
'key': self._api_key,
'alt': 'json' if use_json else 'protojson',
}
logger.debug("Fetching '{}' with '{}'".format(url, body_json))
res = yield from http_utils.fetch(
'post', url, headers=headers, cookies=cookies, params=params,
data=json.dumps(body_json), connector=self._connector
)
logger.debug('Response to request for {} was {}:\n{}'
.format(endpoint, res.code, res.body))
return res
def _request(self, endpoint, body_json):
"""Make chat API request."""
url = 'https://clients6.google.com/chat/v1/{}'.format(endpoint)
headers = {
'authorization': self._get_authorization_header(),
'x-origin': ORIGIN_URL,
'x-goog-authuser': '******',
'content-type': 'application/json+protobuf',
}
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {cookie: self._get_cookie(cookie)
for cookie in required_cookies}
params = {
'key': self._api_key,
'alt': 'json', # json or protojson
}
res = yield http_utils.fetch(
url, method='POST', headers=headers, cookies=cookies,
params=params, data=json.dumps(body_json),
request_timeout=REQUEST_TIMEOUT, connect_timeout=CONNECT_TIMEOUT
)
logger.debug('Response to request for {} was {}:\n{}'
.format(endpoint, res.code, res.body))
if res.code != 200:
raise ValueError('Request to {} endpoint failed with {}: {}'
.format(endpoint, res.code, res.body.decode()))
return res
def _base_request(self, url, content_type, response_type, data):
"""Send a generic authenticated POST request.
Args:
url (str): URL of request.
content_type (str): Request content type.
response_type (str): The desired response format. Valid options
are: 'json' (JSON), 'protojson' (pblite), and 'proto' (binary
Protocol Buffer). 'proto' requires manually setting an extra
header 'X-Goog-Encode-Response-If-Executable: base64'.
data (str): Request body data.
Returns:
FetchResponse: Response containing HTTP code, cookies, and body.
Raises:
NetworkError: If the request fails.
"""
sapisid_cookie = self._get_cookie('SAPISID')
headers = channel.get_authorization_headers(sapisid_cookie)
headers['content-type'] = content_type
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {cookie: self._get_cookie(cookie)
for cookie in required_cookies}
params = {
# "alternative representation type" (desired response format).
'alt': response_type,
}
res = yield from http_utils.fetch(
'post', url, headers=headers, cookies=cookies, params=params,
data=data, connector=self._connector
)
return res
def _fetch_channel_sid(self):
"""Request a new session ID for the push channel.
Raises hangups.NetworkError.
"""
logger.info('Requesting new session...')
url = 'https://talkgadget.google.com{}bind'.format(self._channel_path)
params = {
'VER': 8,
'clid': self._clid_param,
'ec': self._ec_param,
'RID': 81187,
# Required if we want our client to be called "AChromeExtension":
'prop': self._prop_param,
}
try:
res = yield from http_utils.fetch(
'post', url, cookies=self._cookies, params=params,
data='count=0', connector=self._connector
)
except exceptions.NetworkError as e:
raise exceptions.HangupsError('Failed to request SID: {}'.format(e))
# TODO: Re-write the function we're calling here to use a schema so we
# can easily catch its failure.
self._sid_param, self._email, self._header_client, self._gsessionid_param = (
_parse_sid_response(res.body)
)
logger.info('New SID: {}'.format(self._sid_param))
logger.info('New email: {}'.format(self._email))
logger.info('New client: {}'.format(self._header_client))
logger.info('New gsessionid: {}'.format(self._gsessionid_param))
def _fetch_channel_sid(self):
"""Creates a new channel for receiving push data.
Raises hangups.NetworkError if the channel can not be created.
"""
logger.info('Requesting new gsessionid and SID...')
# There's a separate API to get the gsessionid alone that Hangouts for
# Chrome uses, but if we don't send a gsessionid with this request, it
# will return a gsessionid as well as the SID.
res = yield from http_utils.fetch(
'post',
CHANNEL_URL_PREFIX.format('channel/bind'),
cookies=self._cookies,
data='count=0',
connector=self._connector,
headers=get_authorization_headers(self._cookies['SAPISID']),
params={
'VER': 8,
'RID': 81187,
'ctype': 'hangouts', # client type
})
self._sid_param, self._gsessionid_param = _parse_sid_response(res.body)
self._is_subscribed = False
logger.info('New SID: {}'.format(self._sid_param))
logger.info('New gsessionid: {}'.format(self._gsessionid_param))
def _subscribe(self):
"""Subscribes the channel to receive relevant events.
Only needs to be called when a new channel (SID/gsessionid) is opened.
"""
# XXX: Temporary workaround for #58
yield from asyncio.sleep(1)
logger.info('Subscribing channel...')
timestamp = str(int(time.time() * 1000))
# Hangouts for Chrome splits this over 2 requests, but it's possible to
# do everything in one.
yield from http_utils.fetch(
'post',
CHANNEL_URL_PREFIX.format('channel/bind'),
cookies=self._cookies,
connector=self._connector,
headers=get_authorization_headers(self._cookies['SAPISID']),
params={
'VER': 8,
'RID': 81188,
'ctype': 'hangouts', # client type
'gsessionid': self._gsessionid_param,
'SID': self._sid_param,
},
data={
'count':
3,
'ofs':
0,
'req0_p': ('{"1":{"1":{"1":{"1":3,"2":2}},"2":{"1":{"1":3,"2":'
'2},"2":"","3":"JS","4":"lcsclient"},"3":' +
timestamp + ',"4":0,"5":"c1"},"2":{}}'),
'req1_p':
('{"1":{"1":{"1":{"1":3,"2":2}},"2":{"1":{"1":3,"2":'
'2},"2":"","3":"JS","4":"lcsclient"},"3":' + timestamp +
',"4":' + timestamp + ',"5":"c3"},"3":{"1":{"1":"babel"}}}'),
'req2_p': ('{"1":{"1":{"1":{"1":3,"2":2}},"2":{"1":{"1":3,"2":'
'2},"2":"","3":"JS","4":"lcsclient"},"3":' +
timestamp + ',"4":' + timestamp +
',"5":"c4"},"3":{"1":{"1":"hangout_invite"}}}'),
},
)
logger.info('Channel is now subscribed')
self._is_subscribed = True
def upload_images(self, links):
"""Download images and upload them to Google+"""
image_id_list = []
for link in links:
# Download image
try:
res = yield from http_utils.fetch('get', link)
except hangups.NetworkError as e:
print('Failed to download image: {}'.format(e))
continue
# Upload image and get image_id
try:
image_id = yield from self._client.upload_image(
io.BytesIO(res.body), filename=os.path.basename(link))
image_id_list.append(image_id)
except hangups.NetworkError as e:
print('Failed to upload image: {}'.format(e))
continue
return image_id_list
def upload_images(self, links):
"""Download images and upload them to Google+"""
image_id_list = []
for link in links:
# Download image
try:
res = yield from http_utils.fetch('get', link)
except hangups.NetworkError as e:
print('Failed to download image: {}'.format(e))
continue
# Upload image and get image_id
try:
image_id = yield from self._client.upload_image(io.BytesIO(res.body),
filename=os.path.basename(link))
image_id_list.append(image_id)
except hangups.NetworkError as e:
print('Failed to upload image: {}'.format(e))
continue
return image_id_list
def _subscribe(self):
"""Subscribes the channel to receive relevant events.
Only needs to be called when a new channel (SID/gsessionid) is opened.
"""
# XXX: Temporary workaround for #58
yield from asyncio.sleep(1)
logger.info('Subscribing channel...')
timestamp = str(int(time.time() * 1000))
# Hangouts for Chrome splits this over 2 requests, but it's possible to
# do everything in one.
yield from http_utils.fetch(
'post', CHANNEL_URL_PREFIX.format('channel/bind'),
cookies=self._cookies, connector=self._connector,
headers=get_authorization_headers(self._cookies['SAPISID']),
params={
'VER': 8,
'RID': 81188,
'ctype': 'hangouts', # client type
'gsessionid': self._gsessionid_param,
'SID': self._sid_param,
},
data={
'count': 3,
'ofs': 0,
'req0_p': ('{"1":{"1":{"1":{"1":3,"2":2}},"2":{"1":{"1":3,"2":'
'2},"2":"","3":"JS","4":"lcsclient"},"3":' +
timestamp + ',"4":0,"5":"c1"},"2":{}}'),
'req1_p': ('{"1":{"1":{"1":{"1":3,"2":2}},"2":{"1":{"1":3,"2":'
'2},"2":"","3":"JS","4":"lcsclient"},"3":' +
timestamp + ',"4":' + timestamp +
',"5":"c3"},"3":{"1":{"1":"babel"}}}'),
'req2_p': ('{"1":{"1":{"1":{"1":3,"2":2}},"2":{"1":{"1":3,"2":'
'2},"2":"","3":"JS","4":"lcsclient"},"3":' +
timestamp + ',"4":' + timestamp +
',"5":"c4"},"3":{"1":{"1":"hangout_invite"}}}'),
},
)
logger.info('Channel is now subscribed')
self._is_subscribed = True
def _base_request(self, url, content_type, data, use_json=True):
"""Make API request.
Raises hangups.NetworkError if the request fails.
"""
headers = channel.get_authorization_headers(self._get_cookie('SAPISID'))
headers['content-type'] = content_type
required_cookies = ['SAPISID', 'HSID', 'SSID', 'APISID', 'SID']
cookies = {cookie: self._get_cookie(cookie)
for cookie in required_cookies}
params = {
'key': self._api_key,
'alt': 'json' if use_json else 'protojson',
}
res = yield from http_utils.fetch(
'post', url, headers=headers, cookies=cookies, params=params,
data=data, connector=self._connector
)
logger.debug('Response to request for {} was {}:\n{}'
.format(url, res.code, res.body))
return res
def send_maps(self, map_list):
"""Sends a request to the server containing maps (dicts)."""
params = {
'VER': 8, # channel protocol version
'RID': 81188, # request identifier
'ctype': 'hangouts', # client type
}
if self._gsessionid_param is not None:
params['gsessionid'] = self._gsessionid_param
if self._sid_param is not None:
params['SID'] = self._sid_param
data_dict = dict(count=len(map_list), ofs=0)
for map_num, map_ in enumerate(map_list):
for map_key, map_val in map_.items():
data_dict['req{}_{}'.format(map_num, map_key)] = map_val
res = yield from http_utils.fetch(
'post', CHANNEL_URL_PREFIX.format('channel/bind'),
cookies=self._cookies, connector=self._connector,
headers=get_authorization_headers(self._cookies['SAPISID']),
params=params, data=data_dict,
)
return res
def _fetch_channel_sid(self):
"""Creates a new channel for receiving push data.
Raises hangups.NetworkError if the channel can not be created.
"""
logger.info('Requesting new gsessionid and SID...')
# There's a separate API to get the gsessionid alone that Hangouts for
# Chrome uses, but if we don't send a gsessionid with this request, it
# will return a gsessionid as well as the SID.
res = yield from http_utils.fetch(
'post', CHANNEL_URL_PREFIX.format('channel/bind'),
cookies=self._cookies, data='count=0', connector=self._connector,
headers=get_authorization_headers(self._cookies['SAPISID']),
params={
'VER': 8,
'RID': 81187,
'ctype': 'hangouts', # client type
}
)
self._sid_param, self._gsessionid_param = _parse_sid_response(res.body)
self._is_subscribed = False
logger.info('New SID: {}'.format(self._sid_param))
logger.info('New gsessionid: {}'.format(self._gsessionid_param))
def _base_request(self, url, content_type, response_type, data):
"""Send a generic authenticated POST request.
Args:
url (str): URL of request.
content_type (str): Request content type.
response_type (str): The desired response format. Valid options
are: 'json' (JSON), 'protojson' (pblite), and 'proto' (binary
Protocol Buffer). 'proto' requires manually setting an extra
header 'X-Goog-Encode-Response-If-Executable: base64'.
data (str): Request body data.
Returns:
FetchResponse: Response containing HTTP code, cookies, and body.
Raises:
NetworkError: If the request fails.
"""
sapisid_cookie = self._get_cookie('SAPISID')
headers = channel.get_authorization_headers(sapisid_cookie)
headers['content-type'] = content_type
# This header is required for Protocol Buffer responses, which causes
# them to be base64 encoded:
headers['X-Goog-Encode-Response-If-Executable'] = 'base64'
params = {
# "alternative representation type" (desired response format).
'alt': response_type,
# API key (required to avoid 403 Forbidden "Daily Limit for
# Unauthenticated Use Exceeded. Continued use requires signup").
'key': API_KEY,
}
res = yield from http_utils.fetch(
self._session, 'post', url, headers=headers, params=params,
data=data
)
return res
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
# We first need to fetch the 'pvt' token, which is required for the
# initialization request (otherwise it will return 400).
try:
res = yield from http_utils.fetch(
'get', PVT_TOKEN_URL, cookies=self._cookies,
connector=self._connector
)
CHAT_INIT_PARAMS['pvt'] = javascript.loads(res.body.decode())[1]
logger.info('Found PVT token: {}'.format(CHAT_INIT_PARAMS['pvt']))
except (exceptions.NetworkError, ValueError) as e:
raise exceptions.HangupsError('Failed to fetch PVT token: {}'
.format(e))
# Now make the actual initialization request:
try:
res = yield from http_utils.fetch(
'get', CHAT_INIT_URL, cookies=self._cookies,
params=CHAT_INIT_PARAMS, connector=self._connector
)
except exceptions.NetworkError as e:
raise exceptions.HangupsError('Initialize chat request failed: {}'
.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
logger.debug("Attempting to load javascript: {}..."
.format(repr(data[:100])))
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
try:
data = data.replace("data:function(){return", "data:")
data = data.replace("}}", "}")
data = javascript.loads(data)
data_dict[data['key']] = data['data']
except ValueError as e:
raise
# logger.debug('Failed to parse initialize chat object: {}\n{}'
# .format(e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._email = data_dict['ds:34'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
_sync_timestamp = parsers.from_timestamp(
# cgserp?
# data_dict['ds:21'][0][1][4]
# data_dict['ds:35'][0][1][4]
data_dict['ds:21'][0][1][4]
)
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
# cgsirp?
# data_dict['ds:20'][0]
# data_dict['ds:35'][0]
data_dict['ds:20'][0]
).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
# csrcrp?
# data_dict['ds:19'][0][3]
# data_dict['ds:36'][0][3]
data_dict['ds:19'][0][3]
)
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
# cgserp?
# data_dict['ds:21'][0]
# data_dict['ds:37'][0]
data_dict['ds:21'][0]
)
except ValueError as e:
logger.warning('Failed to parse initial client entities: {}'
.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity,
entities.group3.entity, entities.group4.entity,
entities.group5.entity
))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
try:
res = yield from http_utils.fetch(
'get', CHAT_INIT_URL, cookies=self._cookies,
params=CHAT_INIT_PARAMS, connector=self._connector
)
except exceptions.NetworkError as e:
raise exceptions.HangupsError('Initialize chat request failed: {}'
.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
logger.debug('Failed to parse initialize chat object: {}\n{}'
.format(e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
_sync_timestamp = parsers.from_timestamp(
data_dict['ds:21'][0][1][4]
)
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
data_dict['ds:20'][0]
).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
data_dict['ds:19'][0][3]
)
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
data_dict['ds:21'][0]
)
except ValueError as e:
logger.warning('Failed to parse initial client entities: {}'
.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity,
entities.group3.entity, entities.group4.entity,
entities.group5.entity
))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
try:
res = yield from http_utils.fetch('get',
CHAT_INIT_URL,
cookies=self._cookies,
params=CHAT_INIT_PARAMS,
connector=self._connector)
except exceptions.NetworkError as e:
raise exceptions.HangupsError(
'Initialize chat request failed: {}'.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
logger.debug(
'Failed to parse initialize chat object: {}\n{}'.format(
e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
_sync_timestamp = parsers.from_timestamp(
data_dict['ds:21'][0][1][4])
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
data_dict['ds:20'][0]).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
data_dict['ds:19'][0][3])
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
data_dict['ds:21'][0])
except ValueError as e:
logger.warning(
'Failed to parse initial client entities: {}'.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity, entities.
group3.entity, entities.group4.entity, entities.group5.entity))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
def _init_talkgadget_1(self):
"""Make first talkgadget request and parse response.
The response body is a HTML document containing a series of script tags
containing JavaScript object. We need to parse the object to get at the
data.
"""
url = 'https://talkgadget.google.com/u/0/talkgadget/_/chat'
params = {
'prop': 'aChromeExtension',
'fid': 'gtn-roster-iframe-id',
'ec': '["ci:ec",true,true,false]',
}
headers = {
# appears to require a browser user agent
'user-agent':
('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
'(KHTML, like Gecko) Chrome/34.0.1847.132 Safari/537.36'),
}
res = yield http_utils.fetch(url,
cookies=self._cookies,
params=params,
headers=headers,
connect_timeout=CONNECT_TIMEOUT,
request_timeout=REQUEST_TIMEOUT)
logger.debug('First talkgadget request result:\n{}'.format(res.body))
if res.code != 200:
raise ValueError(
"First talkgadget request failed with {}: {}".format(
res.code, res.body))
res = res.body.decode()
# Parse the response by using a regex to find all the JS objects, and
# parsing them.
res = res.replace('\n', '')
regex = re.compile(
r"(?:<script>AF_initDataCallback\((.*?)\);</script>)")
data_dict = {}
for data in regex.findall(res):
try:
data = javascript.loads(data)
data_dict[data['key']] = data['data']
except ValueError as e:
# not everything will be parsable, but we don't care
logger.debug('Failed to parse JavaScript: {}\n{}'.format(
e, data))
# TODO: handle errors here
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
# build dict of conversations and their participants
initial_conversations = {}
self.initial_users = {} # {UserID: User}
# add self to the contacts
self_contact = data_dict['ds:20'][0][2]
self.self_user_id = UserID(chat_id=self_contact[8][0],
gaia_id=self_contact[8][1])
self.initial_users[self.self_user_id] = User(
id_=self.self_user_id,
full_name=self_contact[9][1],
first_name=self_contact[9][2],
is_self=True)
conversations = data_dict['ds:19'][0][3]
for c in conversations:
id_ = c[1][0][0]
participants = c[1][13]
last_modified = c[1][3][12]
# With every converstion, we get a list of up to 20 of the most
# recent messages, sorted oldest to newest.
messages = []
for raw_message in c[2]:
message = longpoll._parse_chat_message([raw_message])
# A message may parse to None if it's just a conversation name
# change.
if message is not None:
messages.append(message[1:])
initial_conversations[id_] = {
'participants': [],
'last_modified': last_modified,
'name': c[1][2],
'messages': messages,
}
# Add the participants for this conversation.
for p in participants:
user_id = UserID(chat_id=p[0][0], gaia_id=p[0][1])
initial_conversations[id_]['participants'].append(user_id)
# Add the participant to our list of contacts as a fallback, in
# case they can't be found later by other methods.
# TODO We should note who these users are and try to request
# them.
# p[1] can be a full name, None, or out of range.
try:
display_name = p[1]
except IndexError:
display_name = None
if display_name is None:
display_name = 'Unknown'
self.initial_users[user_id] = User(
id_=user_id,
first_name=display_name.split()[0],
full_name=display_name,
is_self=(user_id == self.self_user_id))
# build dict of contacts and their names (doesn't include users not in
# contacts)
contacts_main = data_dict['ds:21'][0]
# contacts_main[2] has some, but the format is slightly different
contacts = (contacts_main[4][2] + contacts_main[5][2] +
contacts_main[6][2] + contacts_main[7][2] +
contacts_main[8][2])
for c in contacts:
user_id = UserID(chat_id=c[0][8][0], gaia_id=c[0][8][1])
self.initial_users[user_id] = User(
id_=user_id,
full_name=c[0][9][1],
first_name=c[0][9][2],
is_self=(user_id == self.self_user_id))
# Create a dict of the known conversations.
self.initial_conversations = {
conv_id: Conversation(
self,
conv_id,
[
self.initial_users[user_id]
for user_id in conv_info['participants']
],
conv_info['last_modified'],
conv_info['name'],
conv_info['messages'],
)
for conv_id, conv_info in initial_conversations.items()
}
def _init_talkgadget_1(self):
"""Make first talkgadget request and parse response.
The response body is a HTML document containing a series of script tags
containing JavaScript object. We need to parse the object to get at the
data.
"""
url = 'https://talkgadget.google.com/u/0/talkgadget/_/chat'
params = {
'prop': 'aChromeExtension',
'fid': 'gtn-roster-iframe-id',
'ec': '["ci:ec",true,true,false]',
}
headers = {
# appears to require a browser user agent
'user-agent': (
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
'(KHTML, like Gecko) Chrome/34.0.1847.132 Safari/537.36'
),
}
res = yield http_utils.fetch(
url, cookies=self._cookies, params=params, headers=headers,
connect_timeout=CONNECT_TIMEOUT, request_timeout=REQUEST_TIMEOUT
)
logger.debug('First talkgadget request result:\n{}'.format(res.body))
if res.code != 200:
raise ValueError("First talkgadget request failed with {}: {}"
.format(res.code, res.body))
res = res.body.decode()
# Parse the response by using a regex to find all the JS objects, and
# parsing them.
res = res.replace('\n', '')
regex = re.compile(
r"(?:<script>AF_initDataCallback\((.*?)\);</script>)"
)
data_dict = {}
for data in regex.findall(res):
try:
data = javascript.loads(data)
data_dict[data['key']] = data['data']
except ValueError as e:
# not everything will be parsable, but we don't care
logger.debug('Failed to parse JavaScript: {}\n{}'
.format(e, data))
# TODO: handle errors here
self._api_key = data_dict['ds:7'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
self._channel_path = data_dict['ds:4'][0][1]
self._clid = data_dict['ds:4'][0][7]
self._channel_ec_param = data_dict['ds:4'][0][4]
self._channel_prop_param = data_dict['ds:4'][0][5]
# build dict of conversations and their participants
initial_conversations = {}
self.initial_users = {} # {UserID: User}
# add self to the contacts
self_contact = data_dict['ds:20'][0][2]
self.self_user_id = UserID(chat_id=self_contact[8][0],
gaia_id=self_contact[8][1])
self.initial_users[self.self_user_id] = User(
id_=self.self_user_id, full_name=self_contact[9][1],
first_name=self_contact[9][2], is_self=True
)
conversations = data_dict['ds:19'][0][3]
for c in conversations:
id_ = c[1][0][0]
participants = c[1][13]
last_modified = c[1][3][12]
initial_conversations[id_] = {
'participants': [],
'last_modified': last_modified,
}
for p in participants:
user_id = UserID(chat_id=p[0][0], gaia_id=p[0][1])
initial_conversations[id_]['participants'].append(
user_id
)
# Add the user to our list of contacts if their name is
# present. This is a hack to deal with some contacts not being
# found via the other methods.
# TODO We should note who these users are and try to request
# them.
if len(p) > 1:
display_name = p[1]
self.initial_users[user_id] = User(
id_=user_id, first_name=display_name.split()[0],
full_name=display_name,
is_self=(user_id == self.self_user_id)
)
# build dict of contacts and their names (doesn't include users not in
# contacts)
contacts_main = data_dict['ds:21'][0]
# contacts_main[2] has some, but the format is slightly different
contacts = (contacts_main[4][2] + contacts_main[5][2] +
contacts_main[6][2] + contacts_main[7][2] +
contacts_main[8][2])
for c in contacts:
user_id = UserID(chat_id=c[0][8][0], gaia_id=c[0][8][1])
self.initial_users[user_id] = User(
id_=user_id, full_name=c[0][9][1], first_name=c[0][9][2],
is_self=(user_id == self.self_user_id)
)
# Create a dict of the known conversations.
self.initial_conversations = {conv_id: Conversation(
self, conv_id, [self.initial_users[user_id] for user_id
in conv_info['participants']],
conv_info['last_modified'],
) for conv_id, conv_info in initial_conversations.items()}
def _initialize_chat(self):
"""Request push channel creation and initial chat data.
Returns instance of InitialData.
The response body is a HTML document containing a series of script tags
containing JavaScript objects. We need to parse the objects to get at
the data.
"""
# We first need to fetch the 'pvt' token, which is required for the
# initialization request (otherwise it will return 400).
try:
res = yield from http_utils.fetch('get',
PVT_TOKEN_URL,
cookies=self._cookies,
connector=self._connector)
CHAT_INIT_PARAMS['pvt'] = javascript.loads(res.body.decode())[1]
logger.info('Found PVT token: {}'.format(CHAT_INIT_PARAMS['pvt']))
except (exceptions.NetworkError, ValueError) as e:
raise exceptions.HangupsError(
'Failed to fetch PVT token: {}'.format(e))
# Now make the actual initialization request:
try:
res = yield from http_utils.fetch('get',
CHAT_INIT_URL,
cookies=self._cookies,
params=CHAT_INIT_PARAMS,
connector=self._connector)
except exceptions.NetworkError as e:
raise exceptions.HangupsError(
'Initialize chat request failed: {}'.format(e))
# Parse the response by using a regex to find all the JS objects, and
# parsing them. Not everything will be parsable, but we don't care if
# an object we don't need can't be parsed.
data_dict = {}
for data in CHAT_INIT_REGEX.findall(res.body.decode()):
try:
logger.debug("Attempting to load javascript: {}...".format(
repr(data[:100])))
data = javascript.loads(data)
# pylint: disable=invalid-sequence-index
data_dict[data['key']] = data['data']
except ValueError as e:
try:
data = data.replace("data:function(){return", "data:")
data = data.replace("}}", "}")
data = javascript.loads(data)
data_dict[data['key']] = data['data']
except ValueError as e:
raise
# logger.debug('Failed to parse initialize chat object: {}\n{}'
# .format(e, data))
# Extract various values that we will need.
try:
self._api_key = data_dict['ds:7'][0][2]
self._email = data_dict['ds:34'][0][2]
self._header_date = data_dict['ds:2'][0][4]
self._header_version = data_dict['ds:2'][0][6]
self._header_id = data_dict['ds:4'][0][7]
_sync_timestamp = parsers.from_timestamp(
# cgserp?
# data_dict['ds:21'][0][1][4]
# data_dict['ds:35'][0][1][4]
data_dict['ds:21'][0][1][4])
except KeyError as e:
raise exceptions.HangupsError('Failed to get initialize chat '
'value: {}'.format(e))
# Parse the entity representing the current user.
self_entity = schemas.CLIENT_GET_SELF_INFO_RESPONSE.parse(
# cgsirp?
# data_dict['ds:20'][0]
# data_dict['ds:35'][0]
data_dict['ds:20'][0]).self_entity
# Parse every existing conversation's state, including participants.
initial_conv_states = schemas.CLIENT_CONVERSATION_STATE_LIST.parse(
# csrcrp?
# data_dict['ds:19'][0][3]
# data_dict['ds:36'][0][3]
data_dict['ds:19'][0][3])
initial_conv_parts = []
for conv_state in initial_conv_states:
initial_conv_parts.extend(conv_state.conversation.participant_data)
# Parse the entities for the user's contacts (doesn't include users not
# in contacts). If this fails, continue without the rest of the
# entities.
initial_entities = []
try:
entities = schemas.INITIAL_CLIENT_ENTITIES.parse(
# cgserp?
# data_dict['ds:21'][0]
# data_dict['ds:37'][0]
data_dict['ds:21'][0])
except ValueError as e:
logger.warning(
'Failed to parse initial client entities: {}'.format(e))
else:
initial_entities.extend(entities.entities)
initial_entities.extend(e.entity for e in itertools.chain(
entities.group1.entity, entities.group2.entity, entities.
group3.entity, entities.group4.entity, entities.group5.entity))
return InitialData(initial_conv_states, self_entity, initial_entities,
initial_conv_parts, _sync_timestamp)
