def __init__(self,
host,
user,
keyfile,
python_executable='~/anaconda3/bin/python'):
self.remote_machine = _get_machine(host, user, keyfile)
self._kill_py()
self.py = self.remote_machine[python_executable]
self.proc = self.py.popen(['-m', 'bnb.remote.server'],
new_session=True)
self.local_port = None
line = ""
try:
logger.debug(f'Waiting for the line')
line = self.proc.stdout.readline()
self.remote_port = int(line.strip())
except Exception:
stdout, stderr = self.proc.communicate()
self.close()
raise ProcessExecutionError(self.proc.argv, self.proc.returncode,
BYTES_LITERAL(line) + stdout, stderr)
def __init__(self,
remote_machine,
server_class="rpyc.utils.server.ThreadedServer",
extra_setup="",
python_executable=None):
self.proc = None
self.tun = None
self.remote_machine = remote_machine
self._tmpdir_ctx = None
rpyc_root = local.path(rpyc.__file__).up()
self._tmpdir_ctx = remote_machine.tempdir()
tmp = self._tmpdir_ctx.__enter__()
copy(rpyc_root, tmp / "rpyc")
script = (tmp / "deployed-rpyc.py")
modname, clsname = server_class.rsplit(".", 1)
script.write(
SERVER_SCRIPT.replace("$MODULE$", modname).replace(
"$SERVER$", clsname).replace("$EXTRA_SETUP$", extra_setup))
if python_executable:
cmd = remote_machine[python_executable]
else:
major = sys.version_info[0]
minor = sys.version_info[1]
cmd = None
for opt in [
"python%s.%s" % (major, minor),
"python%s" % (major, )
]:
try:
cmd = remote_machine[opt]
except CommandNotFound:
pass
else:
break
if not cmd:
cmd = remote_machine.python
self.proc = cmd.popen(script, new_session=True)
line = ""
try:
line = self.proc.stdout.readline()
self.remote_port = int(line.strip())
except Exception:
try:
self.proc.terminate()
except Exception:
pass
stdout, stderr = self.proc.communicate()
raise ProcessExecutionError(self.proc.argv, self.proc.returncode,
BYTES_LITERAL(line) + stdout, stderr)
if hasattr(remote_machine, "connect_sock"):
# Paramiko: use connect_sock() instead of tunnels
self.local_port = None
else:
self.local_port = rpyc.utils.factory._get_free_port()
self.tun = remote_machine.tunnel(self.local_port, self.remote_port)
def __init__(self, remote_machine, server_class="ThreadedServer"):
self.proc = None
self.tun = None
self._tmpdir_ctx = None
rpyc_root = local.path(rpyc.__file__).dirname
self._tmpdir_ctx = remote_machine.tempdir()
tmp = self._tmpdir_ctx.__enter__()
copy(rpyc_root, tmp)
script = (tmp / "deployed-rpyc.py")
script.write(SERVER_SCRIPT.replace("$SERVER$", server_class))
self.proc = remote_machine.python.popen(script, new_session=True)
line = ""
try:
line = self.proc.stdout.readline()
remote_port = int(line.strip())
except Exception:
try:
self.proc.terminate()
except Exception:
pass
stdout, stderr = self.proc.communicate()
raise ProcessExecutionError(self.proc.argv, self.proc.returncode,
line + stdout, stderr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
self.local_port = s.getsockname()[1]
s.close()
self.tun = remote_machine.tunnel(self.local_port, remote_port)
def __init__(self,
remote_machine,
server_class="rpyc.utils.server.ThreadedServer",
extra_setup=""):
self.proc = None
self.tun = None
self.remote_machine = remote_machine
self._tmpdir_ctx = None
rpyc_root = local.path(rpyc.__file__).up(2)
self._tmpdir_ctx = remote_machine.tempdir()
tmp = self._tmpdir_ctx.__enter__()
copy(rpyc_root, tmp)
script = (tmp / "deployed-rpyc.py")
modname, clsname = server_class.rsplit(".", 1)
script.write(
SERVER_SCRIPT.replace("$MODULE$", modname).replace(
"$SERVER$", clsname).replace("$EXTRA_SETUP$", extra_setup))
self.proc = remote_machine.python.popen(script, new_session=True)
line = ""
try:
line = self.proc.stdout.readline()
self.remote_port = int(line.strip())
except Exception:
try:
self.proc.terminate()
except Exception:
pass
stdout, stderr = self.proc.communicate()
raise ProcessExecutionError(self.proc.argv, self.proc.returncode,
line + stdout, stderr)
if hasattr(remote_machine, "connect_sock"):
# Paramiko: use connect_sock() instead of tunnels
self.local_port = None
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
self.local_port = s.getsockname()[1]
s.close()
self.tun = remote_machine.tunnel(self.local_port, self.remote_port)
def deployment(remote_machine, code=SERVER_SCRIPT):
with remote_machine.tempdir() as tmp:
copy(RPYC_ROOT, tmp)
delete(tmp // ".pyc", tmp // "*/.pyc")
with remote_machine.cwd(tmp):
with remote_machine.env(
PYTHONPATH=remote_machine.env.get("PYTHONPATH", "") +
":%s" % (tmp, )):
proc = (remote_machine.python << code).popen()
line = ""
try:
line = proc.stdout.readline()
remote_port = int(line.strip())
except Exception:
try:
proc.kill()
except Exception:
pass
stdout, stderr = proc.communicate()
raise ProcessExecutionError(proc.argv, proc.returncode,
line + stdout, stderr)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("localhost", 0))
local_port = s.getsockname()[1]
s.close()
with remote_machine.tunnel(local_port, remote_port) as tun:
try:
yield DeployedServer(local_port)
finally:
try:
proc.kill()
except Exception:
pass
def download_url_and_send(self,
bot,
url,
direct_urls,
chat_id,
reply_to_message_id=None,
wait_message_id=None):
bot.send_chat_action(chat_id=chat_id, action=ChatAction.RECORD_AUDIO)
download_dir = os.path.join(self.DL_DIR, str(uuid4()))
shutil.rmtree(download_dir, ignore_errors=True)
os.makedirs(download_dir)
status = 0
if direct_urls == "direct":
status = -3
elif direct_urls == "country":
status = -4
elif direct_urls == "live":
status = -5
else:
if (self.SITES["sc"] in url
and self.SITES["scapi"] not in url) or (self.SITES["bc"]
in url):
cmd_name = "scdl"
cmd_args = []
cmd = None
cmd_input = None
if self.SITES["sc"] in url and self.SITES["scapi"] not in url:
cmd_name = "scdl"
cmd_args = (
"-l",
url, # URL of track/playlist/user
"-c", # Continue if a music already exist
"--path",
download_dir, # Download the music to a custom path
"--onlymp3", # Download only the mp3 file even if the track is Downloadable
"--addtofile", # Add the artist name to the filename if it isn't in the filename already
"--addtimestamp", # Adds the timestamp of the creation of the track to the title (useful to sort chronologically)
"--no-playlist-folder",
# Download playlist tracks into directory, instead of making a playlist subfolder
)
cmd = scdl_bin
cmd_input = None
elif self.SITES["bc"] in url:
cmd_name = "bandcamp-dl"
cmd_args = (
"--base-dir",
download_dir, # Base location of which all files are downloaded
"--template",
"%{track} - %{artist} - %{title} [%{album}]", # Output filename template
"--overwrite", # Overwrite tracks that already exist
"--group", # Use album/track Label as iTunes grouping
"--embed-art", # Embed album art (if available)
"--no-slugify", # Disable slugification of track, album, and artist names
url, # URL of album/track
)
cmd = bandcamp_dl_bin
cmd_input = "yes"
logger.info("%s starts: %s", cmd_name, url)
cmd_proc = cmd[cmd_args].popen(stdin=PIPE,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True)
try:
cmd_stdout, cmd_stderr = cmd_proc.communicate(
input=cmd_input, timeout=self.DL_TIMEOUT)
cmd_retcode = cmd_proc.returncode
if cmd_retcode or "Error resolving url" in cmd_stderr:
raise ProcessExecutionError(cmd_args, cmd_retcode,
cmd_stdout, cmd_stderr)
logger.info("%s succeeded: %s", cmd_name, url)
status = 1
except TimeoutExpired:
cmd_proc.kill()
logger.info("%s took too much time and dropped: %s", url)
status = -1
except ProcessExecutionError:
logger.exception("%s failed: %s" % (cmd_name, url))
if status == 0:
cmd_name = "youtube-dl"
cmd = youtube_dl_func
# TODO: set different ydl_opts for different sites
ydl_opts = {
'format':
'bestaudio/best',
'outtmpl':
os.path.join(download_dir, '%(title)s.%(ext)s'),
# default: %(autonumber)s - %(title)s-%(id)s.%(ext)s
'postprocessors': [
{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '128',
},
# {'key': 'EmbedThumbnail',}, {'key': 'FFmpegMetadata',},
],
}
queue = Queue()
cmd_args = (
url,
ydl_opts,
queue,
)
logger.info("%s starts: %s", cmd_name, url)
cmd_proc = Process(target=cmd, args=cmd_args)
cmd_proc.start()
try:
cmd_retcode, cmd_stderr = queue.get(block=True,
timeout=self.DL_TIMEOUT)
cmd_stdout = ""
cmd_proc.join()
if cmd_retcode:
raise ProcessExecutionError(cmd_args, cmd_retcode,
cmd_stdout, cmd_stderr)
# raise cmd_status #TODO: pass and re-raise original Exception?
logger.info("%s succeeded: %s", cmd_name, url)
status = 1
except Empty:
cmd_proc.join(1)
if cmd_proc.is_alive():
cmd_proc.terminate()
logger.info("%s took too much time and dropped: %s", cmd_name,
url)
status = -1
except ProcessExecutionError:
logger.exception("%s failed: %s" % (cmd_name, url))
status = -2
gc.collect()
if status == -1:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.DL_TIMEOUT_TEXT,
parse_mode='Markdown')
elif status == -2:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.NO_AUDIO_TEXT,
parse_mode='Markdown')
elif status == -3:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.DIRECT_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == -4:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.REGION_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == -5:
bot.send_message(chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=self.LIVE_RESTRICTION_TEXT,
parse_mode='Markdown')
elif status == 1:
file_list = []
for d, dirs, files in os.walk(download_dir):
for file in files:
file_list.append(os.path.join(d, file))
for file in sorted(file_list):
file_name = os.path.split(file)[-1]
file_parts = []
try:
file_parts = self.split_audio_file(file)
except FileNotSupportedError as exc:
if not (exc.file_format in ["m3u", "jpg", "jpeg", "png"]):
logger.warning("Unsupported file format: %s",
file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, downloaded file `{}` is in format I could not yet convert or send"
.format(file_name),
parse_mode='Markdown')
except FileTooLargeError as exc:
logger.info("Large file for convert: %s", file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, downloaded file `{}` is `{}` MB and it is larger than I could convert (`{} MB`)"
.format(file_name, exc.file_size // 1000000,
self.MAX_CONVERT_FILE_SIZE // 1000000),
parse_mode='Markdown')
except FileConvertedPartiallyError as exc:
file_parts = exc.file_parts
logger.exception("Pydub failed: %s" % file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text="*Sorry*, not enough memory to convert file `{}`.."
.format(file_name),
parse_mode='Markdown')
except FileNotConvertedError as exc:
logger.exception("Pydub failed: %s" % file_name)
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text="*Sorry*, not enough memory to convert file `{}`.."
.format(file_name),
parse_mode='Markdown')
try:
caption = "Downloaded from {} with @{}\n".format(
URL(url).host, self.bot_username)
if "qP303vxTLS8" in url:
caption += "\n" + random.choice([
"Скачал музла, машина эмпэтри дала!",
"У тебя талант, братан! Ка-какой? Качать онлайн!",
"Слушаю и не плачУ, то, что скачал вчера",
"Всё по чесноку, если скачал, отгружу музла!",
"Дёрнул за канат, и телега поймала трэкан!",
"Сегодня я качаю, и трэки не влазят мне в RAM!",
])
flood = self.chat_storage[str(
chat_id)]["settings"]["flood"]
sent_audio_ids = self.send_audio_file_parts(
bot, chat_id, file_parts,
reply_to_message_id if flood == "yes" else None,
caption if flood == "yes" else None)
except FileSentPartiallyError as exc:
sent_audio_ids = exc.sent_audio_ids
bot.send_message(
chat_id=chat_id,
reply_to_message_id=reply_to_message_id,
text=
"*Sorry*, could not send file `{}` or some of it's parts.."
.format(file_name),
parse_mode='Markdown')
logger.warning("Sending some parts failed: %s" % file_name)
if not self.SERVE_AUDIO:
shutil.rmtree(download_dir, ignore_errors=True)
if wait_message_id: # TODO: delete only once
try:
bot.delete_message(chat_id=chat_id, message_id=wait_message_id)
except:
pass
def __call__(self):
raise ProcessExecutionError([], 1, "", "")