async def on_message(event): if event.raw_text == '/start': return l.debug(event) # BOT_AGENT_CHAT_ID - client user id to bypass 50 MB limit if event.from_id == int(os.getenv('BOT_AGENT_CHAT_ID')): await share_content_with_user(event) return try: idlog = _log.new_logger(user_id=event.from_id) if event.from_id in in_progress_users or event.from_id in pending_torrents: #user = await client.get_entity(event.from_id) #idlog.info(user.username + ' ' + 'sent message while' + # ' ' + ('being in progress' if event.from_id in in_progress_users else 'in pending state')) idlog.debug('in_progress_users ' + str(in_progress_users)) idlog.debug('pending_torrents ' + str(pending_torrents)) await event.reply('Wait until current torrent will be downloaded') return magnet_link_or_bytes = await get_torrent_from_event(idlog, event) if magnet_link_or_bytes is not None: # set dumb data until torrent will be resolved pending_torrents[event.from_id] = True else: return e = await event.reply('Resolving torrent, please wait...') th = None try: th = await get_torrent_handle(idlog, magnet_link_or_bytes) except NoMetadataError: await e.edit( 'Couldn\'t resolve magnet link for metadata, torrent was removed from downloads. Sorry' ) del pending_torrents[event.from_id] return except Exception as err: idlog.exception(err) await e.edit('Error occured') del pending_torrents[event.from_id] return nidlog = _log.new_logger(torrent_name=th.name(), user_id=event.from_id) nidlog.debug('Torrent handler created successfull') s = th.status() files_info = th.files() files = [File(f) for f in files_info] nidlog.debug(files_info) files_size_sum = files_size(files) nidlog.info('Content size: ' + sizeof_fmt(files_size_sum)) if files_size_sum > 20 * 1024 * 1024 * 1024: await e.edit('Torrent is too big, max allowed size is 20 GB') del pending_torrents[event.from_id] session.remove_torrent(th) return buttons = [] if files_size_sum <= 5 * 1024 * 1024 * 1024: buttons.append( Button.inline('over Telegram(slow)', '1:' + str(event.message.id))) # buttons.append(Button.inline('via Web(Zip)', '2:'+str(event.message.id))) # if len(files) <= MAX_LEN_FILES_FOR_RAW: buttons.append(Button.inline('over Web', '3:' + str(event.message.id))) # status_msg_text, status_msg_current_file_text, zip_progress_text = prepare_status_message(files, s.name, th.total_size()) zip_progress_text = prepare_status_message(files, s.name, th.total_size()) status_msg = await e.edit( zip_progress_text.format(0).format(progress=""), buttons=buttons) zfile = prepare_zip_file(th, s.name, files, status_msg, nidlog) #zfile.progress_text = zip_progress_text pending_torrents[event.from_id] = ((zfile, event.message.id), time.time()) # if filze size < 2 GB allow upload via Telegram # otherwise only web allowed # if files_size_sum <= 2*1024*1024*1024 or len(files) <= MAX_LEN_FILES_FOR_RAW: # nidlog.debug('zfile.size = ', zfile.size) # pending_torrents[event.from_id] = ((zfile, event.message.id), time.time()) # return # Upload via Web without buttons # try: # nidlog.info('Uploading via web') # await status_msg.edit(status_msg.message, buttons=[Button.inline('Cancel', str(event.from_id))]) # in_progress_users.add(event.from_id) # del pending_torrents[event.from_id] # zfile.set_should_split(False) # zfile.event = status_msg # fu = await fex.FexUploader.new() # upload_task = client.loop.create_task(fu.add_file(zfile.name, zfile.size, zfile)) # tasks[event.from_id] = upload_task # try: # await upload_task # except asyncio.CancelledError as e: # print(e) # #await fu.add_file(zfile.name, zfile.size, zfile) # await fu.delete() # if not upload_task.cancelled(): # await status_msg.edit(zfile.progress_text.format(zfile.last_percent),buttons=[Button.url('Download content', fu.download_link)]) # # #await upload_to_ipfs(zfile) # except Exception as e: # print(e) # traceback.print_exc() # finally: # if event.from_id in tasks: # del tasks[event.from_id] # zfile.close() # session.remove_torrent(zfile.torrent_handler) # in_progress_users.remove(event.from_id) except Exception as e: l.exception(e)
async def on_button(event): global session try: l.debug(event) button_id = None zfile = None idlog = _log.new_logger(user_id=event.sender_id) #user = await client.get_entity(event.sender_id) if event.sender_id in in_progress_users: #idlog.info(user.username + ' ' + 'Button pressed while' + # ' ' + ('being in progress' if event.sender_id in in_progress_users else 'in pending state')) idlog.debug('in_progress_users ' + str(in_progress_users)) idlog.debug('pending_torrents ' + str(pending_torrents)) await event.reply('Wait until current torrent will be downloaded') return in_progress_users.add(event.sender_id) nidlog = None if event.sender_id not in pending_torrents or int( event.data.split(b':')[1]) != pending_torrents[ event.sender_id][0][1]: # handle torrent request from old message idlog.info('request torrent handle from old message') event.message = await (await event.get_message()).get_reply_message() idlog.debug('reply message ' + str(event.message)) magnet_link_or_bytes = await get_torrent_from_event(idlog, event) if magnet_link_or_bytes is None: idlog.warning('Not magnet or torrent in old message') try: in_progress_users.remove(event.sender_id) await event.reply('Failed resolve torrent') except: pass return th = None try: th = await get_torrent_handle(idlog, magnet_link_or_bytes) except NoMetadataError: await event.edit( 'Couldn\'t resolve magnet link for metadata, torrent was removed from downloads. Sorry' ) del pending_torrents[event.sender_id] in_progress_users.remove(event.sender_id) return except Exception as e: idlog.exception(e) await event.edit('Error occured') del pending_torrents[event.sender_id] in_progress_users.remove(event.sender_id) return th.calc_prioritized_piece_count() th.prepare_pieces_priority() nidlog = _log.new_logger(torrent_name=th.name(), user_id=event.sender_id) s = th.status() files_info = th.files() files = [File(f) for f in files_info] nidlog.debug(files_info) #status_msg_text, status_msg_current_file_text, zips = prepare_status_message(files, s.name, th.total_size()) # zfile = prepare_zip_file(th, s.name, files, event, status_msg_text, status_msg_current_file_text, event, nidlog) zfile = prepare_zip_file(th, s.name, files, event, nidlog) #zfile.progress_text = zips button_id, _ = event.data.split(b':') elif event.sender_id in pending_torrents: zfile, origin_msg_id = pending_torrents[event.sender_id][0] nidlog = _log.new_logger(torrent_name=zfile.torrent_handler.name(), user_id=event.sender_id) del pending_torrents[event.sender_id] button_id, msg_id = event.data.split(b':') #if int(msg_id) != origin_msg_id: # button was pressed on old torrent when new torrent was added recently # nidlog.info('button was pressed on old torrent when new torrent was added recently') # session.remove_torrent(zfile.torrent_handler) if button_id == b'1': # via Telegram nidlog.info('via Telegram') try: # await event.edit((await event.get_message()).message, buttons=[Button.inline('Cancel', str(event.sender_id))]) await zfile.progress_callback(0) # await event.edit((await event.get_message()).message) zfile.event = event upload_task = client.loop.create_task( upload_all_torrent_content(zfile, event, nidlog)) tasks[event.sender_id] = upload_task try: await upload_task except asyncio.CancelledError as e: nidlog.info('Canceled') if not upload_task.cancelled(): # TODO receive progress_text somehow another way nidlog.info('Completed!') _files = [File(f) for f in zfile.torrent_handler.files()] progress_text = prepare_status_message( _files, zfile.torrent_handler.name(), zfile.torrent_handler.total_size()) await event.edit(progress_text.format(100)) # size = zfile.size # f = open('test.zip', 'wb') # d = await zfile.read(500*1024) # while len(d) > 0: # f.write(d) # await asyncio.sleep(0.1) # d = await zfile.read(500*1024) # f.close() except tc.NoActivityTimeoutError: await event.edit( 'Couldn\'t find any peers, torrent was removed from downloads. Sorry' ) except Exception as e: nidlog.exception(e) finally: zfile.close() session.remove_torrent(zfile.torrent_handler) elif button_id == b'2': # via Web nidlog.info('via Web') try: # url = 'https://api.anonfiles.com/upload' # url = 'http://127.0.0.1' # r = await httpx.post(url=url, files={'file': SyncZipTorrentContentFile(zfile)}) # print(r.text) # curl = await asyncio.create_subprocess_exec('/usr/bin/curl', # '-T', # '-', # '-X', # 'POST', # 'https://api.anonfiles.com/upload', # stdout=asyncio.subprocess.PIPE, # stdin=asyncio.subprocess.PIPE, loop=client.loop) # await asyncio.sleep(5) # d = await zfile.read() # while d: # print('curl write ', len(d)) # curl.stdin.write(d) # await curl.stdin.drain() # d = await zfile.read() # print('curl close stdin') # curl.stdin.close() # ret = await curl.stdout.read() # print(ret) # curl.kill() # headers={ # 'Content-Length': str(zfile.size + 195 + 2*len(zfile.name.encode('utf'))) # } # data = aiohttp.FormData(quote_fields=False) # data.add_field('file', zfile, filename=zfile.name) # async with aiohttp.ClientSession() as ses: # print('begin file uploading') # async with ses.post(url, data=data, headers=headers) as resp: # print(await resp.text()) await event.edit( (await event.get_message()).message, buttons=[Button.inline('Cancel', str(event.sender_id))]) zfile.set_should_split(False) fu = await fex.FexUploader.new() upload_task = client.loop.create_task( fu.add_file(zfile.name, zfile.size, zfile)) tasks[event.sender_id] = upload_task try: await upload_task except asyncio.CancelledError as e: nidlog.info('Canceled') # await fu.add_file(zfile.name, zfile.size, zfile) if not upload_task.cancelled(): nidlog.info('Download url ' + fu.download_link) # TODO receive progress_text somehow another way _files = [File(f) for f in zfile.torrent_handler.files()] progress_text = prepare_status_message( _files, zfile.torrent_handler.name(), zfile.torrent_handler.total_size()) await event.edit(progress_text.format(100), buttons=[ Button.url('Download content', fu.download_link) ]) # await upload_to_ipfs(zfile) except tc.NoActivityTimeoutError: await event.edit( 'Couldn\'t find any peers, torrent was removed from downloads. Sorry' ) except Exception as e: nidlog.exception(e) finally: await fu.delete() zfile.close() session.remove_torrent(zfile.torrent_handler) # await event.edit((await event.get_message()).message,buttons=[Button.url('Download content', 'http://206.189.63.205:8080/'+str(file_key))]) elif button_id == b'3': # upload via Web raw nidlog.info('via Web raw') try: fu = await fex.FexUploader.new(nidlog) await event.edit( (await event.get_message()).message, buttons=[Button.inline('Cancel', str(event.sender_id))]) files = [] relative_size = 0 _files = [File(f) for f in zfile.torrent_handler.files()] progress_text = prepare_status_message( _files, zfile.torrent_handler.name(), zfile.torrent_handler.total_size()) callback = lambda percent: \ event.edit(progress_text.format(percent), buttons=[Button.inline('Cancel', str(event.sender_id))]) for f in zfile.files: files.append( fex.FexFile( tc.AsyncTorrentContentFileWrapper( f, callback, relative_size, zfile.files_size_sum, nidlog), f.info.fullpath, f.info.size)) relative_size += f.info.size upload_task = client.loop.create_task(fu.upload_files(files)) tasks[event.sender_id] = upload_task try: await upload_task except asyncio.CancelledError as e: nidlog.info('Canceled') if not upload_task.cancelled(): nidlog.info('Completed!') nidlog.info('Download url ' + fu.download_link) await event.edit(progress_text.format(100), buttons=[ Button.url('Download content', fu.download_link) ]) except tc.NoActivityTimeoutError: await event.edit( 'Couldn\'t find any peers, torrent was removed from downloads. Sorry' ) except Exception as e: nidlog.exception(e) finally: await fu.delete() zfile.close() session.remove_torrent(zfile.torrent_handler) else: nidlog.error('Error: wrong button id ', button_id) except Exception as e: l.exception(e) finally: if event.sender_id in tasks: if not tasks[event.sender_id].cancelled(): tasks[event.sender_id].cancel() del tasks[event.sender_id] if event.sender_id in in_progress_users: in_progress_users.remove(event.sender_id) if event.sender_id in pending_torrents: del pending_torrents[event.sender_id]
import fex import torrent_content as tc import const bot_token = os.getenv('BOT_TOKEN') api_id = int(os.getenv('API_ID')) api_hash = os.getenv('API_HASH') client = TelegramClient('toby', api_id, api_hash) bot = TelegramClient('bot', api_id, api_hash).start(bot_token=bot_token) session = None in_progress_users = set() tasks = dict() pending_torrents = dict() l = _log.new_logger() class File(): def __init__(self, file_info): self.fullpath = file_info[0] self.num_pieces = file_info[1][0] self.size = file_info[1][1] @property def name(self): return os.path.basename(self.fullpath) def files_size(files): files_size_sum = 0
async def _on_message(event): if event.raw_text == '/start': return l.debug(event) # BOT_AGENT_CHAT_ID - client user id to bypass 50 MB limit if event.from_id == int(os.getenv('BOT_AGENT_CHAT_ID')): await share_content_with_user(event) return idlog = _log.new_logger(user_id=event.from_id) if event.from_id in in_progress_users or event.from_id in pending_torrents: #user = await client.get_entity(event.from_id) #idlog.info(user.username + ' ' + 'sent message while' + # ' ' + ('being in progress' if event.from_id in in_progress_users else 'in pending state')) idlog.debug('in_progress_users ' + str(in_progress_users)) idlog.debug('pending_torrents ' + str(pending_torrents)) await event.reply('Wait until current torrent will be downloaded') return magnet_link_or_bytes = await get_torrent_from_event(idlog, event) if magnet_link_or_bytes is not None: # set dumb data until torrent will be resolved pending_torrents[event.from_id] = True else: return e = await event.reply('Resolving torrent, please wait...') th = None try: session = session_manager.new_session(event.from_id) th = await get_torrent_handle(idlog, magnet_link_or_bytes, session) except NoMetadataError as err: idlog.error(err) await e.edit( 'Couldn\'t resolve magnet link for metadata, torrent was removed from downloads. Sorry' ) raise except Exception as err: idlog.exception(err) await e.edit('Error occured') raise nidlog = _log.new_logger(torrent_name=th.name(), user_id=event.from_id) nidlog.debug('Torrent handler created successfull') s = th.status() files_info = th.files() files = [File(f) for f in files_info] nidlog.debug(files_info) files_size_sum = files_size(files) nidlog.info('Content size: ' + sizeof_fmt(files_size_sum)) if files_size_sum > 20 * 1024 * 1024 * 1024: session.remove_torrent(th) await e.edit('ERROR: too big torrent.\n' 'For Web max allowed torrent size 20 GB,\n' 'individual file size should be less then 10 GB.\n' 'For Telegram max allowed torrent size 10 GB') raise Exception('Too big torrent') buttons = [] if files_size_sum <= 10 * 1024 * 1024 * 1024: buttons.append([ Button.inline('over Telegram(zip)', '1:' + str(event.message.id)) ]) is_raw_tg_suitable = True for f in files: if f.size > 1500 * 1024 * 1024: is_raw_tg_suitable = False break if is_raw_tg_suitable: buttons.append( [Button.inline('over Telegram', '2:' + str(event.message.id))]) # buttons.append(Button.inline('via Web(Zip)', '4:'+str(event.message.id))) # if len(files) <= MAX_LEN_FILES_FOR_RAW: if max(files, key=lambda f: f.size).size < 10 * 1024 * 1024 * 1024: buttons.append( [Button.inline('over Web', '3:' + str(event.message.id))]) if len(buttons) == 0: nidlog.info('Too big torrent size content') await e.edit('ERROR: too big torrent.\n' 'For Web max allowed torrent size 20 GB,\n' 'individual file size should be less then 10 GB.\n' 'For Telegram max allowed torrent size 10 GB') session.remove_torrent(th) raise Exception('Too big torrent') # status_msg_text, status_msg_current_file_text, zip_progress_text = prepare_status_message(files, s.name, th.total_size()) zip_progress_text = prepare_status_message(files, s.name, th.total_size()) status_msg = await e.edit(zip_progress_text.format(0).format(progress=""), buttons=buttons) zfile = prepare_zip_file(th, s.name, files, status_msg, nidlog) #zfile.progress_text = zip_progress_text pending_torrents[event.from_id] = ((zfile, event.message.id), time.time())
#TODO # Error handling # EBS Support import sys sys.path.append(".") sys.path.append("lib") import time from boto.pyami.config import Config from boto.ec2.connection import EC2Connection import logger import getopt import ConfigParser SSH_OPTS = '-o StrictHostKeyChecking=no -i ' log = logger.new_logger("deployEC2") class ManageEC2(object): instance_ids = [] def __init__(self): self.aws_access_key_id = None self.aws_secret_access_key = None self.ami = None self.key_name = None self.instance_type = None self.zone = None self.security_groups = None self.os = None self.num_nodes = None self.ssh_username = None
#TODO # Error handling # EBS Support import sys sys.path.append(".") sys.path.append("lib") import time from boto.pyami.config import Config from boto.ec2.connection import EC2Connection import logger import getopt import ConfigParser SSH_OPTS = '-o StrictHostKeyChecking=no -i ' log = logger.new_logger("deployEC2") class ManageEC2(object): instance_ids = [] def __init__(self): self.aws_access_key_id = None self.aws_secret_access_key = None self.ami = None self.key_name = None self.instance_type = None self.zone = None self.security_groups = None self.os = None self.num_nodes = None