def unpack_message(session, reader):
"""Unpacks a message following MtProto 2.0 guidelines"""
# See https://core.telegram.org/mtproto/description
if reader.read_long(signed=False) != session.auth_key.key_id:
raise SecurityError('Server replied with an invalid auth key')
msg_key = reader.read(16)
aes_key, aes_iv = calc_key(session.auth_key.key, msg_key, False)
data = BinaryReader(AES.decrypt_ige(reader.read(), aes_key, aes_iv))
data.read_long() # remote_salt
if data.read_long() != session.id:
raise SecurityError('Server replied with a wrong session ID')
remote_msg_id = data.read_long()
remote_sequence = data.read_int()
msg_len = data.read_int()
message = data.read(msg_len)
# https://core.telegram.org/mtproto/security_guidelines
# Sections "checking sha256 hash" and "message length"
if msg_key != sha256(session.auth_key.key[96:96 + 32] +
data.get_bytes()).digest()[8:24]:
raise SecurityError("Received msg_key doesn't match with expected one")
return message, remote_msg_id, remote_sequence
def unpack_message(session, reader):
"""Unpacks a message following MtProto 2.0 guidelines"""
# See https://core.telegram.org/mtproto/description
if reader.read_long(signed=False) != session.auth_key.key_id:
raise SecurityError('Server replied with an invalid auth key')
msg_key = reader.read(16)
aes_key, aes_iv = calc_key(session.auth_key.key, msg_key, False)
data = BinaryReader(AES.decrypt_ige(reader.read(), aes_key, aes_iv))
data.read_long() # remote_salt
if data.read_long() != session.id:
raise SecurityError('Server replied with a wrong session ID')
remote_msg_id = data.read_long()
remote_sequence = data.read_int()
msg_len = data.read_int()
message = data.read(msg_len)
# https://core.telegram.org/mtproto/security_guidelines
# Sections "checking sha256 hash" and "message length"
if msg_key != sha256(
session.auth_key.key[96:96 + 32] + data.get_bytes()).digest()[8:24]:
raise SecurityError("Received msg_key doesn't match with expected one")
return message, remote_msg_id, remote_sequence
def test_binary_tgwriter_tgreader(self):
small_data = os.urandom(33)
small_data_padded = os.urandom(19) # +1 byte for length = 20 (%4 = 0)
large_data = os.urandom(999)
large_data_padded = os.urandom(1024)
data = (small_data, small_data_padded, large_data, large_data_padded)
string = 'Testing Telegram strings, this should work properly!'
serialized = b''.join(TLObject.serialize_bytes(d) for d in data) + \
TLObject.serialize_bytes(string)
with BinaryReader(serialized) as reader:
# And then try reading it without errors (it should be unharmed!)
for datum in data:
value = reader.tgread_bytes()
self.assertEqual(
value,
datum,
msg='Example bytes should be {} but is {}'.format(
datum, value))
value = reader.tgread_string()
self.assertEqual(
value,
string,
msg='Example string should be {} but is {}'.format(
string, value))
def test_binary_writer_reader():
# Test that we can read properly
data = b'\x01\x05\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00' \
b'\x88A\x00\x00\x00\x00\x00\x009@\x1a\x1b\x1c\x1d\x1e\x1f ' \
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' \
b'\x00\x80'
with BinaryReader(data) as reader:
value = reader.read_byte()
assert value == 1, 'Example byte should be 1 but is {}'.format(
value)
value = reader.read_int()
assert value == 5, 'Example integer should be 5 but is {}'.format(
value)
value = reader.read_long()
assert value == 13, 'Example long integer should be 13 but is {}'.format(
value)
value = reader.read_float()
assert value == 17.0, 'Example float should be 17.0 but is {}'.format(
value)
value = reader.read_double()
assert value == 25.0, 'Example double should be 25.0 but is {}'.format(
value)
value = reader.read(7)
assert value == bytes([26, 27, 28, 29, 30, 31, 32]), 'Example bytes should be {} but is {}' \
.format(bytes([26, 27, 28, 29, 30, 31, 32]), value)
value = reader.read_large_int(128, signed=False)
assert value == 2**127, 'Example large integer should be {} but is {}'.format(
2**127, value)
def convert_vector(blob):
"""Converts an sql blob back to vector of TLObjects"""
if not blob:
return []
with BinaryReader(blob) as reader:
return reader.tgread_vector()
def convert_object(blob):
"""Converts an sql blob back to a TLObject"""
if not blob:
return None
with BinaryReader(blob) as reader:
try:
return reader.tgread_object()
except TypeNotFoundError:
return None
async def _handle_gzip_packed(self, message):
"""
Unpacks the data from a gzipped object and processes it:
gzip_packed#3072cfa1 packed_data:bytes = Object;
"""
self._log.debug('Handling gzipped data')
with BinaryReader(message.obj.data) as reader:
message.obj = reader.tgread_object()
await self._process_message(message)
async def _handle_rpc_result(self, message):
"""
Handles the result for Remote Procedure Calls:
rpc_result#f35c6d01 req_msg_id:long result:bytes = RpcResult;
This is where the future results for sent requests are set.
"""
rpc_result = message.obj
state = self._pending_state.pop(rpc_result.req_msg_id, None)
self._log.debug('Handling RPC result for message %d',
rpc_result.req_msg_id)
if not state:
# TODO We should not get responses to things we never sent
# However receiving a File() with empty bytes is "common".
# See #658, #759 and #958. They seem to happen in a container
# which contain the real response right after.
try:
with BinaryReader(rpc_result.body) as reader:
if not isinstance(reader.tgread_object(), upload.File):
raise ValueError('Not an upload.File')
except (TypeNotFoundError, ValueError):
self._log.info('Received response without parent request: %s',
rpc_result.body)
return
if rpc_result.error:
error = rpc_message_to_error(rpc_result.error, state.request)
self._send_queue.append(
RequestState(MsgsAck([state.msg_id]), loop=self._loop))
if not state.future.cancelled():
state.future.set_exception(error)
else:
with BinaryReader(rpc_result.body) as reader:
result = state.request.read_result(reader)
if not state.future.cancelled():
state.future.set_result(result)
async def random_shout(self, peer):
chat_id = telethon.utils.get_peer_id(peer)
row = await self.pool.fetchrow(self.queries.random_shout(), chat_id)
if row is None:
return None
message_id, content, encoded_entities = row
entities = [
BinaryReader(encoded).tgread_object()
for encoded in encoded_entities
]
return types.Message(id=message_id,
to_id=chat_id,
message=content,
entities=entities)
def fetch_dialogs(self, cache_file='dialogs.tl', force=False):
"""Get a list of dialogs, and dump new data from them"""
# TODO What to do about cache invalidation?
if not force and os.path.isfile(cache_file):
with open(cache_file, 'rb') as f, BinaryReader(stream=f) as reader:
entities = []
while True:
try:
entities.append(reader.tgread_object())
except BufferError:
break # No more data left to read
return entities
with open(cache_file, 'wb') as f:
entities = [d.entity for d in self.client.get_dialogs(limit=None)]
for entity in entities:
f.write(bytes(entity))
return entities
def decrypt_mtproto1(self, message_key, chat_id, encrypted_data):
aes_key, aes_iv = _old_calc_key(
self.get_secret_chat(chat_id).auth_key, message_key, True)
decrypted_data = AES.decrypt_ige(encrypted_data, aes_key, aes_iv)
message_data_length = struct.unpack('<I', decrypted_data[:4])[0]
message_data = decrypted_data[4:message_data_length + 4]
if message_data_length > len(decrypted_data):
raise SecurityError("message data length is too big")
if message_key != sha1(
decrypted_data[:4 + message_data_length]).digest()[-16:]:
raise SecurityError("Message key mismatch")
if len(decrypted_data) - 4 - message_data_length > 15:
raise SecurityError("Difference is too big")
if len(decrypted_data) % 16 != 0:
raise SecurityError("Decrypted data can not be divided by 16")
return BinaryReader(message_data).tgread_object()
def enumerate_backups_entities():
"""Enumerates the entities of all the available backups"""
if isdir(Backuper.backups_dir):
# Look for subdirectories
for directory in listdir(Backuper.backups_dir):
entity_file = path.join(Backuper.backups_dir, directory,
'entity.tlo')
# Ensure the entity.pickle file exists
if isfile(entity_file):
# Load and yield it
with open(entity_file, 'rb') as file:
with BinaryReader(stream=file) as reader:
try:
yield reader.tgread_object()
except TypeNotFoundError:
# Old user, scheme got updated, don't care.
pass
def decrypt_mtproto2(self, message_key, chat_id, encrypted_data):
peer = self.get_secret_chat(chat_id)
aes_key, aes_iv = MTProtoState._calc_key(peer.auth_key, message_key,
not peer.admin)
decrypted_data = AES.decrypt_ige(encrypted_data, aes_key, aes_iv)
message_data_length = struct.unpack('<I', decrypted_data[:4])[0]
message_data = decrypted_data[4:message_data_length + 4]
if message_data_length > len(decrypted_data):
raise SecurityError("message data length is too big")
is_admin = (8 if peer.admin else 0)
first_str = peer.auth_key[88 + is_admin:88 + 32 + is_admin]
if message_key != sha256(first_str + decrypted_data).digest()[8:24]:
raise SecurityError("Message key mismatch")
if len(decrypted_data) - 4 - message_data_length < 12:
raise SecurityError("Padding is too small")
if len(decrypted_data) % 16 != 0:
raise SecurityError("Decrpyted data not divisble by 16")
return BinaryReader(message_data).tgread_object()
async def submit(
self,
request: SubmitRequestPb,
context: ServicerContext,
metadata: dict,
) -> SubmitResponsePb:
session_id = metadata.get('session-id')
request_context = RequestContext(
bot_name=self.service_name,
chat=request.chat,
request_id=metadata.get('request-id'),
)
request_context.add_default_fields(
mode='submit',
session_id=metadata.get('session-id'),
**self.get_default_service_fields(),
)
document = BinaryReader(request.telegram_document).tgread_object()
if document.size > 20 * 1024 * 1024:
request_context.error_log(FileTooBigError(size=document.size))
request_context.statbox(action='file_too_big')
await self.telegram_client.send_message(
request_context.chat.chat_id,
t('FILE_TOO_BIG_ERROR',
language=request_context.chat.language),
buttons=[close_button()],
)
return SubmitResponsePb()
processing_message = await self.telegram_client.send_message(
request_context.chat.chat_id,
t("PROCESSING_PAPER",
language=request_context.chat.language).format(
filename=document.attributes[0].file_name, ),
)
try:
file = await self.telegram_client.download_document(
document=document, file=bytes)
try:
processed_document = await self.grobid_client.process_fulltext_document(
pdf_file=file)
except BadRequestError as e:
request_context.statbox(action='unparsable_document')
request_context.error_log(e)
await self.telegram_client.send_message(
request_context.chat.chat_id,
t('UNPARSABLE_DOCUMENT_ERROR',
language=request_context.chat.language).format(
filename=document.attributes[0].file_name, ),
buttons=[close_button()],
)
return SubmitResponsePb()
if not processed_document.get('doi'):
request_context.statbox(action='unparsable_doi')
request_context.error_log(UnparsableDoiError())
await self.telegram_client.send_message(
request_context.chat.chat_id,
t('UNPARSABLE_DOI_ERROR',
language=request_context.chat.language).format(
filename=document.attributes[0].file_name, ),
buttons=[close_button()],
)
return SubmitResponsePb()
search_response_pb = await self.meta_api_client.search(
schemas=('scimag', ),
query=processed_document['doi'],
page=0,
page_size=1,
request_id=request_context.request_id,
session_id=session_id,
user_id=str(request_context.chat.chat_id),
language=request_context.chat.language,
)
if len(search_response_pb.scored_documents) == 0:
request_context.statbox(action='unavailable_metadata')
request_context.error_log(
UnavailableMetadataError(doi=processed_document['doi']))
await self.telegram_client.send_message(
request_context.chat.chat_id,
t('UNAVAILABLE_METADATA_ERROR',
language=request_context.chat.language).format(
doi=processed_document['doi']),
buttons=[close_button()],
)
return SubmitResponsePb()
document_view = ScimagView(
search_response_pb.scored_documents[0].typed_document.scimag)
uploaded_message = await self.send_file(
document_view=document_view,
file=file,
request_context=request_context,
session_id=session_id,
voting=False,
)
finally:
await processing_message.delete()
document_operation_pb = DocumentOperationPb(
update_document=UpdateDocumentPb(
typed_document=TypedDocumentPb(sharience=ShariencePb(
parent_id=document_view.id,
uploader_id=request_context.chat.chat_id,
updated_at=int(time.time()),
md5=hashlib.md5(file).hexdigest(),
filesize=document.size,
ipfs_multihashes=await self.get_ipfs_hashes(file=file),
telegram_file_id=uploaded_message.file.id,
)), ), )
request_context.statbox(
action='success',
document_id=document_view.id,
schema='scimag',
)
await operation_log(document_operation_pb)
return SubmitResponsePb()
def deserialize_vector(vector: bytes) -> Sequence[TLObject]:
return BinaryReader(vector).tgread_vector()