def _open_ssh_connection(self):
tries = 5
logging.info("Connecting to instance %s " %
self.instance.public_ip_address)
logging.info("ssh_key: %s " % self.config.ssh_key)
ssh_conn = None
while tries > 0:
try:
ssh_conn = Connection(
host=self.instance.public_ip_address,
user="******",
forward_agent=False,
connect_kwargs={"key_filename": [self.config.ssh_key]},
)
ssh_conn.open()
tries = 0
except BaseException:
logging.info("SSH connection error - retrying...")
tries -= 1
time.sleep(20)
if (ssh_conn is None) or (not ssh_conn.is_connected):
raise ConnectionError()
return ssh_conn
def ssh_connection(testing_container, request):
# Initialize SSH connection to testing Docker container.
connection = Connection('localhost',
user='******',
port=49000,
connect_kwargs={'password': '******'})
connection.open()
return connection
def ssh_con_fabric(test_vars):
"""Create an SSH connection to the controller."""
log = logging.getLogger("ssh_con_fabric")
# SSH connection/client to the public IP.
pub_client = Connection(test_vars["public_ip"],
user=test_vars["controller_user"],
connect_kwargs={
"key_filename": test_vars["ssh_priv_key"],
})
# If the controller's IP is not the same as the public IP, then we are
# using a jumpbox to get into the VNET containing the controller. In that
# case, create an SSH tunnel before connecting to the controller.
msg_con = "SSH connection to controller ({})".format(
test_vars["controller_ip"])
if test_vars["public_ip"] != test_vars["controller_ip"]:
for port_attempt in range(1, 11):
tunnel_local_port = get_unused_local_port()
tunnel_remote_port = 22
msg_con += " via jumpbox ({0}), local port {1}".format(
test_vars["public_ip"], tunnel_local_port)
log.debug("Opening {}".format(msg_con))
with pub_client.forward_local(
local_port=tunnel_local_port,
remote_port=tunnel_remote_port,
remote_host=test_vars["controller_ip"]):
client = Connection("127.0.0.1",
user=test_vars["controller_user"],
port=tunnel_local_port,
connect_kwargs={
"key_filename":
test_vars["ssh_priv_key"],
})
try:
client.open()
except NoValidConnectionsError as ex:
exp_err = "Unable to connect to port {} on 127.0.0.1".format(
tunnel_local_port)
if exp_err not in str(ex):
raise
else:
log.warning("{0} (attempt #{1}, retrying)".format(
exp_err, str(port_attempt)))
continue
yield client
log.debug("{} closed".format(msg_con))
break # no need to iterate again
else:
log.debug("Opening {}".format(msg_con))
pub_client.open()
yield pub_client
log.debug("Closing {}".format(msg_con))
pub_client.close()
def wait_until_running(self, public_ip=None, private_key=None):
if public_ip is None:
public_ip = self.public_ip
if self.cloud_session is None:
raise Exception(
'VM has no session associated. This is almost sure because it has not been '
'successfully instantiated.')
if self.vm_id is None:
raise Exception(
'VM has no id associated. This is almost sure because it has not been '
'successfully instantiated.')
if public_ip is None:
raise Exception(
"VM has no public IP associated. A valid gateway to access to the machine has not been "
"supplied")
one = self.cloud_session.one
user = self.base_user
vm_id = self.vm_id
public_ip = self.public_ip
if private_key is None:
private_key = self.private_key
print("Waiting until vm with id %d and name %s is running" %
(vm_id, one.vmpool.info(-1, vm_id, vm_id, -1).VM[0].NAME),
end="")
# ACTIVE and RUNNING
while not (one.vm.info(vm_id).STATE == 3
and one.vm.info(vm_id).LCM_STATE == 3):
print(".", end="")
sys.stdout.flush()
sleep(1)
print("")
print("Waiting until vm with id %d and name %s has ssh reachable" %
(vm_id, one.vmpool.info(-1, vm_id, vm_id, -1).VM[0].NAME),
end="")
keep_trying = True
connection_args = {}
if private_key is not None:
connection_args["connect_kwargs"] = {"key_filename": private_key}
if user is not None:
connection_args["user"] = user
curr_conn = Connection(public_ip, **connection_args)
while keep_trying:
print(".", end="")
sys.stdout.flush()
try:
curr_conn.open()
curr_conn.close()
keep_trying = False
except paramiko.ssh_exception.NoValidConnectionsError as e:
if "Unable to connect to port 22" not in str(e):
raise e
sleep(1)
print("")
class SshWorker:
"""
Wrap operations at work via ssh
"""
def __init__(self, ip_address, private_key_path, user='******'):
self.ip = ip_address
self.logger = logging.getLogger('ssh_worker')
self.logger.debug(f"Establishing connection to {ip_address}")
self.connection = Connection(
ip_address,
user=user,
connect_kwargs={"key_filename": [private_key_path]})
def __del__(self):
self.logger.debug("Closing SSH connection")
self.connection.close()
def is_alive(self):
retries = 5
while retries > 0:
self.logger.debug(
f"Trying to connect to {self.ip} ({retries} left)")
retries = retries - 1
try:
self.connection.open()
assert self.hostname().ok
except ConnectionRefusedError:
self.logger.info(f"Worker {self.ip} refused connection.")
time.sleep(5)
except AssertionError:
self.logger.info(f"Worker {self.ip} returned invalid output.")
time.sleep(5)
except:
self.logger.info(
f"Worker {self.ip} returned unknown exception.")
time.sleep(5)
return True
def hostname(self):
return self.connection.run('hostname ; whoami ; id')
def bootstrap_host(self):
return self.connection.run(
'export DEBIAN_FRONTEND=noninteractive; '
'apt-get update && '
'apt-get install --no-install-recommends -yy '
'masscan nmap;')
def masscan(self):
return self.connection.run(
'export NOBANNER_TCP_PORTS="[80, 443, 8080]"; '
'masscan -iL /tmp/targets.list'
' --open-only'
' -oJ /tmp/output.json'
' --rate 10000'
' -p 1-65535 -p U:1-65535')
def deploy_node(self):
self.node_count += 1
if self.node_count > MAX_WORKER:
raise Exception("Too many workers")
ip = self.get_node_ip(self.node_count)
name = self.get_node_name(self.node_count)
logging.info("Deploying node {} at {}".format(name, ip))
try:
container = self.client.containers.get(name)
if container.status == "running":
raise Exception(
"container of the same name `{}` is still running".format(
name))
container.remove() # remove name
except:
pass
# start a node in the network
server = self.client.containers.create("broadway-nested",
name=name,
detach=True,
privileged=True)
self.client.networks.get(self.network).connect(server, ipv4_address=ip)
server.start()
conn = Connection(NESTED_USER + "@" + ip,
connect_kwargs={"password": NESTED_PASSWORD})
tries = 0
while tries < MAX_CONN_TRY:
logging.info("Trying to connect to {}".format(ip))
try:
conn.open()
break
except:
pass
tries += 1
if tries >= MAX_CONN_TRY:
raise Exception("Failed to connect to to {}".format(ip))
return conn
def resource_session(request):
"""Pytest fixture that provides instantiated session objects for each
of the classes that inherit the Session or POSIXSession classes.
The fixture will run the test method once for each session object provided.
Parameters
----------
request : object
Pytest request object that contains the class to test against
Yields
-------
session object
Instantiated object based on a class that extends the Session or
POSIXSession class
"""
# Initialize docker connection to testing container.
if request.param in [DockerSession, PTYDockerSession]:
client = docker.Client()
container = [
c for c in client.containers()
if '/testing-container' in c['Names']
][0]
yield request.param(client, container)
elif request.param in [SSHSession, PTYSSHSession]:
# Initialize SSH connection to testing Docker container.
connection = Connection(
'localhost',
user='******',
port=49000,
connect_kwargs={
'password': '******'
}
)
connection.open()
yield request.param(connection)
elif request.param in [SingularitySession, PTYSingularitySession]:
# Initialize Singularity test container.
name = str(uuid.uuid4().hex)[:11]
resource = Singularity(name=name, image='docker://python:2.7')
resource.connect()
resource.create()
yield request.param(name)
resource.delete()
else:
yield request.param()
class SshSession:
def __init__(self, host, user, key=None):
self.host = host
self.port = 22
self.user = user
self.key = key
self.pkey = paramiko.RSAKey.from_private_key(io.StringIO(self.key))
self.connection = Connection(self.host,
user=self.user,
port=22,
connect_kwargs={'pkey': self.pkey})
@retry(tries=5, delay=10)
def connect(self):
self.connection.open()
def run(self, command, **kwargs):
# http://docs.pyinvoke.org/en/latest/api/runners.html#invoke.runners.Runner.run
if not self.connection.is_connected:
self.connect()
self.connection.run(command, hide=True, **kwargs)
def mkdir(self, path, **kwargs):
self.connection.run('mkdir {}'.format(path), **kwargs)
def python(self, code, **kwargs):
self.connection.run('python {}'.format(code), **kwargs)
@contextmanager
def cd(self, *path):
cd_path = '/'.join(list(path))
self.connection.command_cwds.append(cd_path)
try:
yield
finally:
self.connection.command_cwds.pop()
@contextmanager
def activate(self, env):
self.connection.command_prefixes.append(
'conda activate {}'.format(env))
try:
yield
finally:
self.connection.command_prefixes.pop()
class SSH():
"""
SSH context manager for creating an SSH connection.
On enter an SSH connection is attempted every 5 seconds until successful.
An exception is raised after 5 minutes.
On exit the connection is closed.
Arguments:
host: Host to connect to.
user: User to connect with.
private_key: RSA private key.
"""
def __init__(self, host: str, user: str, private_key: RSAKey):
self.host = host
self.user = user
self.private_key = RSAKey.from_private_key(StringIO(private_key))
def __enter__(self):
self.connection = Connection(
host=self.host,
user=self.user,
connect_kwargs={'pkey': self.private_key},
)
logger.info(f'Waiting for SSH to become available on {self.host}...')
self.wait_for_ssh(default_timer())
return self.connection
def __exit__(self, type, value, traceback):
logger.info(f'Closing SSH connection to {self.host}...')
self.connection.close()
def wait_for_ssh(self, start):
try:
self.connection.open()
except NoValidConnectionsError:
# Error after 5 minutes. Otherwise retry.
now = default_timer()
if now - start > 300:
raise
sleep(5)
self.wait_for_ssh(start)
def uploadzip(c):
c = Connection(MYSOC_SERVER,
user=SSH_USER,
port=22,
connect_kwargs={"key_filename": SSH_FILENAME,
"passphrase": SSH_PASSPHRASE})
c.open()
path, filename = os.path.split(bake_zip)
sudo_password = ""
while not sudo_password:
sudo_password = getpass("sudo password: "******"uploading {0}".format(filename))
c.put(bake_zip, filename)
#c.sudo('rm -rf {0}'.format(destination_path))
c.sudo('unzip -o {0} -d {1}'.format(filename, destination_path),
password=sudo_password)
c.close()
def ssh_con_fabric(test_vars):
"""Create an SSH connection to the controller."""
log = logging.getLogger("ssh_con_fabric")
# SSH connection/client to the public IP.
pub_client = Connection(test_vars["public_ip"],
user=test_vars["controller_user"],
connect_kwargs={
"key_filename": test_vars["ssh_priv_key"],
})
# If the controller's IP is not the same as the public IP, then we are
# using a jumpbox to get into the VNET containing the controller. In that
# case, create an SSH tunnel before connecting to the controller.
msg_con = "SSH connection to controller ({})".format(
test_vars["controller_ip"])
if test_vars["public_ip"] != test_vars["controller_ip"]:
tunnel_local_port = get_unused_local_port()
tunnel_remote_port = 22
msg_con += " via jumpbox ({0}), local port {1}".format(
test_vars["public_ip"], tunnel_local_port)
log.debug("Opening {}".format(msg_con))
with pub_client.forward_local(local_port=tunnel_local_port,
remote_port=tunnel_remote_port,
remote_host=test_vars["controller_ip"]):
client = Connection("127.0.0.1",
user=test_vars["controller_user"],
port=tunnel_local_port,
connect_kwargs={
"key_filename": test_vars["ssh_priv_key"],
})
client.open()
yield client
log.debug("{} closed".format(msg_con))
else:
log.debug("Opening {}".format(msg_con))
pub_client.open()
yield pub_client
log.debug("Closing {}".format(msg_con))
pub_client.close()
class SSH(object):
"""
对 Fabric 的一个简单的封装：
1. 屏蔽了一些暂时用不到的参数。
2. 设置了一些对 debug 有利的默认参数
3. 添加了额外的中文 docstring
"""
def __init__(
self,
host: str,
port: int,
username: str,
password: Optional[str] = None,
key_file_obj: IO = None,
key_file_path: Optional[Path] = None,
key_file_passphrase: Optional[str] = None,
):
"""
使用示例：
```python3
# 1. 使用密码登录远程主机
ssh_con = SSH("192.168.1.xxx", 22, username="******", password="******")
# 2. 使用私钥登录远程主机（私钥没有设置 passphrase）
## 2.1 指定密钥位置
ssh_con = SSH("192.168.1.xxx", 22, username="******", key_file_path=Path("~/.ssh/id_rsa"))
## 2.2 给出密钥的 IO 对象
ssh_con = SSH("192.168.1.xxx", 22, username="******", key_file_obj=Path("~/.ssh/id_rsa").open(encoding='utf-8'))
ssh_con = SSH("192.168.1.xxx", 22, username="******", key_file_obj=StringIO("<private-key-content>"))
```
"""
connect_kwargs = dict()
if key_file_obj is not None:
private_key = paramiko.RSAKey.from_private_key(
key_file_obj, key_file_passphrase)
connect_kwargs['pkey'] = private_key
elif key_file_path is not None:
connect_kwargs = {
"key_filename": str(key_file_path.resolve()),
"passphrase": key_file_passphrase
}
elif password is not None:
connect_kwargs['password'] = password
else:
raise KeyError("must given password/pkey/private_key")
self.conn = Connection(host=host,
port=port,
user=username,
connect_kwargs=connect_kwargs)
def open(self):
"""建立连接。
使用 run/put/get 命令时，会自动创建连接。
但是 cd 不行，因为 cd 只是修改本地 session 的东西
"""
return self.conn.open()
def close(self):
"""关闭连接"""
return self.conn.close()
@property
def is_connected(self):
return self.conn.is_connected
def run(self, cmd: str, warn=False, hide=False, echo=True, **kwargs):
"""
远程执行命令
使用示例：
```python3
# 1. 执行命令，打印出被执行的命令，以及命令的输出。命令失败抛出异常
ssh_con.run("ls -al")
# 2. 执行命令，命令失败只抛出 warn（对程序的运行没有影响），这样可以手动处理异常情况。
result = ssh_con.run("ls -al", warn=True)
if result.return_code != 0: # 命名执行失败
# 处理失败的情况
# 3. 拉取 docker 镜像，只在命令失败的情况下，才输出 docker 命令的日志。
result = ssh_con.run("docker pull xxxx", hide=True, warn=True)
if result.return_code != 0: # 运行失败
logger.error(result.stdout) # 打印出错误日志，这里 stdout 一般会包含 stderr
# 然后考虑要不要抛出异常
```
==================
注意！！！run/sudo 并不记录 cd 命令切换的路径！
如果需要改变 self.cwd （当前工作目录），必须使用 self.cd() 函数，详细的用法参见该函数的 docstring
官方文档：http://docs.pyinvoke.org/en/0.12.1/api/runners.html#invoke.runners.Runner.run
:param cmd: 命令字符串
:param warn: 命令非正常结束时，默认抛异常。如果这个为 True，就只发出 Warning，不抛异常
:param hide: 是否隐藏掉命令的输出流（stdout/stderr）
:param echo:是否回显正在运行的命令（最好选择回显，debug很有用）
:param shell: 指定用于执行命令的 shell
:param encoding: 字符集
:return: 一个 Result 对象，该对象的主要参数有：
command: 被执行的命令
ok: A boolean equivalent to exited == 0.
return_code: 命令返回值
stdout: 命令的标准输出，是一个多行字符串
程执行命令时可能无法区分 stdout/stderr，这时 stdout 会包含 stderr
"""
return self.conn.run(command=cmd,
warn=warn,
hide=hide,
echo=echo,
**kwargs)
def sudo(self, command, **kwargs):
"""以 sudo 权限执行命令
如果设置了密码，就自动使用该密码。
否则会要求在命令行输入密码（这对运维来说显然不可取）
注意！！！run/sudo 并不记录 cd 命令切换的路径！
如果需要改变 self.cwd （当前工作目录），必须使用 self.cd() 函数，详细的用法参见该函数的 docstring
"""
return self.conn.sudo(command=command, **kwargs)
def local(self, *args, **kwargs):
"""在本机执行命令"""
return self.conn.local(*args, **kwargs)
def cd(self, path: Union[Path, str]):
"""change dir
self.run()/self.sudo() 命令不会记录由 `cd` 命令造成的工作目录改变，
要使多个语句都在某个指定的路径下执行，就必须使用 self.cd()，
（或者你手动在每个 run 指令前，加上 cd /home/xxx/xxx，显然不推荐这么干）
重点！这是一个类似 open(xxx) 的函数，需要使用 with 做上下文管理。
用法：
```
with ssh_conn.cd("/tmp"):
# do some thing
```
出了这个 with 语句块，cd 就失效了。
---
实际上就是给 with 语句块中的每个 run/sudo 命令，添加上 `cd xxx`
"""
return self.conn.cd(str(path))
@property
def cwd(self):
"""currently work dir
默认为空字符串，表示 $HOME
"""
return self.conn.cwd
def get(self,
remote_file_path: Union[Path, str],
local: Union[Path, IO] = None,
preserve_mode: bool = True,
mkdirs=False):
"""
从远程主机获取文件到本地
:param remote_file_path: 远程主机上的文件的路径（不会解析 `~` 符号！建议用绝对路径！）
:param local: 将文件保存到本地的这个位置/flie-like obj。若未指定，会存放在当前工作目录下(os.getcwd())
:param preserve_mode: 是否保存文件的 mode 信息（可读/可写/可执行），默认 True
:param mkdirs: 如果路径不存在，是否自动创建中间文件夹。
:return: 一个 Result 对象
"""
if isinstance(local, Path):
local_path_parent = local.parent
if local_path_parent.exists() is False:
if mkdirs:
local_path_parent.mkdir(parents=True)
else:
raise FileNotFoundError(
"directory '{}' not exist!".format(local_path_parent))
return self.conn.get(
remote=str(remote_file_path),
local=local,
preserve_mode=preserve_mode,
)
def put(self,
local: Union[Path, IO],
remote_file_path: Union[Path, str] = Path("."),
preserve_mode: bool = True,
mkdirs=False):
"""
将文件从本地传输给远程主机
:param local: 本机的文件路径/ file-like obj
:param remote_file_path: 将文件保存到远程主机上的这个路径下（不会解析 `~` 符号！建议用绝对路径！）
默认传输到远程的 home 目录下
:param preserve_mode: 是否保存文件的 mode 信息（可读/可写/可执行），默认 True
:param mkdirs: 如果路径不存在，是否自动创建中间文件夹。
:return: 一个 Result 对象，该对象不包含 ok 等属性。。。
"""
if mkdirs:
parent = Path(remote_file_path).parent
self.conn.run("mkdir -p '{}'".format(parent))
return self.conn.put(local=local,
remote=Path(remote_file_path).as_posix(),
preserve_mode=preserve_mode)
def put_dir(self,
local_dir_path: Path,
remote_path: Union[Path, str] = Path("."),
preserve_mode: bool = True,
mkdirs=False):
"""
将文件夹从本地传输给远程主机
:param local_dir_path: 本机的文件夹路径
:param remote_path: 远程主机中，已经存在的一个文件夹的路径（不会解析 `~` 符号！建议用绝对路径！）
默认传输到远程的 home 目录下
:param preserve_mode: 是否保存文件的 mode 信息（可读/可写/可执行），默认 True
:param mkdirs: 如果路径不存在，是否自动创建中间文件夹。
:return
"""
try:
self.conn.run(f"test -d {Path(remote_path).as_posix()}")
except UnexpectedExit:
raise RuntimeError(
"remote_path 必须是一个已经存在的文件夹路径！请给定正确的 remote_path，或者使用默认参数！")
stream = tar_files(local_dir_path, c_type="gz", get_stream=True)
tar_name = local_dir_path.resolve().name + ".tar.gz"
stream.name = tar_name
self.put(local=stream,
remote_file_path=Path(remote_path).as_posix(),
preserve_mode=preserve_mode,
mkdirs=mkdirs)
with self.cd(remote_path):
self.run("tar -ax -f {}".format(tar_name))
self.run("rm {}".format(tar_name))
def open_method_generates_real_connection(self):
c = Connection("localhost")
c.open()
assert c.client.get_transport().active is True
assert c.is_connected is True
return c
class SSHKBM(QObject):
def __init__(self, args):
super().__init__()
self.connection = None
self.conn_params = {p: None for p in ('host', 'user', 'port', 'password')}
self.display = None
# Initialise UI
self.app = QApplication(sys.argv)
self.window = QMainWindow()
self.ui = Ui_SSHKBMWindow()
self.ui.setupUi(self.window)
self.on_disconnect()
# Bind events
self.ui.connectButton.clicked.connect(self.click_connect)
self.ui.connectButton.setShortcut('Return')
self.ui.sendTextButton.clicked.connect(self.click_send_text)
QShortcut(QKeySequence('Ctrl+Return'),
self.ui.typingField,
activated=self.click_send_text)
orig_evt = self.ui.keyboardTab.keyPressEvent
def new_evt(evt):
self.keyboard_key_pressed(evt.key(), evt.modifiers())
orig_evt(evt)
self.ui.keyboardTab.keyPressEvent = new_evt
key_buttons = [b + 'Btn' for b in
[d + 'Arrow' for d in ['up', 'down', 'left', 'right']] +
['f' + str(n) for n in range(1, 12)] +
['tab', 'caps', 'num', 'scroll',
'ins', 'del', 'prtscr', 'pgup', 'pgdn', 'home', 'end', 'esc',
'volUp', 'volDown', 'mute', 'play', 'stop', 'prev', 'next', 'space']]
btn_clk = lambda b: lambda: self.keyboard_key_pressed(\
QObject.property(getattr(self.ui, b), 'Key'), 0)
for b in key_buttons:
getattr(self.ui, b).clicked.connect(btn_clk(b))
self.mp = self.ui.mousePicture
def new_evt(evt):
self.mouse_cmd(evt.pos())
self.mp.mousePressEvent = new_evt
# Fill in default values
self.ui.hostField.setText(args.get('host', ''))
self.ui.portField.setText(args.get('port', ''))
self.ui.userField.setText(args.get('user', ''))
self.ui.passwordField.setText(args.get('password', ''))
self.ui.displayField.setText(args.get('display', ''))
# Finally, show the window
self.window.show()
if args.get('connect', False):
self.click_connect()
self.app.exec_()
def on_connect(self, connect=True):
for i in range(1, 4):
self.ui.tabWidget.setTabEnabled(i, connect)
for e in [self.ui.hostField, self.ui.portField, self.ui.userField,
self.ui.passwordField]:
e.setEnabled(not connect)
self.ui.statusbar.showMessage('Connected.' if connect else 'Disconnected.')
self.ui.connectButton.setText('Connect' if not connect else 'Disconnect')
if connect:
LockKeyState(self.connection, onchange=self.update_lock_state)
def on_disconnect(self):
self.on_connect(False)
def _get_connection_params(self):
none_if_empty = lambda s : s if s else None
self.conn_params.update({
'host': none_if_empty(str(self.ui.hostField.text()).strip()),
'port': none_if_empty(str(self.ui.portField.text())),
'user': none_if_empty(str(self.ui.userField.text())),
'password': none_if_empty(str(self.ui.passwordField.text()))
})
def update_lock_state(self, keys):
self.lock_keys = keys
for k in ['Caps', 'Num', 'Scroll']:
font = QFont()
font.setItalic(keys[k])
getattr(self.ui, k.lower() + 'Btn').setFont(font)
@pyqtSlot()
def click_connect(self):
if self.connection is not None and self.connection.is_connected:
self.connection.close()
self.on_connect(self.connection.is_connected)
return
self._get_connection_params()
k = None
if self.conn_params['password'] is not None:
k = {'password': self.conn_params['password']}
if self.conn_params['host'] is None:
QMessageBox.critical(self.ui.centralwidget, 'Error',
'Please fill in the host.', QMessageBox.Ok)
return
self.connection = Connection(
self.conn_params['host'],
port=self.conn_params['port'],
user=self.conn_params['user'],
connect_kwargs = k,
)
self.connection.open()
self.on_connect(self.connection.is_connected)
@pyqtSlot()
def click_send_text(self):
text = str(self.ui.typingField.toPlainText())
display = str(self.ui.displayField.text())
cmd = 'DISPLAY=' + shlex.quote(display) + ' '
cmd += 'xdotool type '
cmd += shlex.quote(text).replace('\n', '\r')
self.connection.run(cmd)
self.ui.typingField.setPlainText('')
def keyboard_key_pressed(self, key, modifiers):
if type(key) == int:
if 0x1001250 <= key <= 0x1001262:
name = characters.DEAD[key]
elif 0x41 <= key <= 0x5a or 0x61 <= key <= 0x7a:
name = chr(key)
else:
name = QKeySequence(key).toString()
try:
name.encode('utf-8')
except UnicodeEncodeError:
return
else:
name = key
k = []
if self.ui.ignoreModifiersCheck.isChecked():
modifiers = 0x0
if modifiers & Qt.KeypadModifier:
name = 'KP_' + name
if self.ui.composeCheck.isChecked():
k.append('Multi_key')
if self.ui.ctrlCheck.isChecked() or (modifiers & Qt.ControlModifier):
k.append('Ctrl')
if self.ui.shiftCheck.isChecked() or (modifiers & Qt.ShiftModifier):
k.append('Shift')
if self.ui.altCheck.isChecked() or (modifiers & Qt.AltModifier):
k.append('Alt')
if self.ui.superCheck.isChecked() or (modifiers & Qt.MetaModifier):
k.append('Super')
if self.ui.altGrCheck.isChecked():
k.append('ISO_Level3_Shift')
if name in characters.CHARACTERS:
name = characters.CHARACTERS[name]
if len(name) == 1:
name = name.lower()
k.append(name)
key_str = '+'.join(k)
self.ui.lastKeyTitleLabel.setText('Last key sent:')
self.ui.lastKeyLabel.setText(key_str)
display = str(self.ui.displayField.text())
cmd = 'DISPLAY=' + shlex.quote(display) + ' '
if self.lock_keys['Num'] and \
((key_str[-1] in digits and key_str[:-1].endswith('KP_')) \
or key_str.endswith('KP_Separator')):
cmd += 'xdotool key Num_Lock ' + shlex.quote(key_str) + ' Num_Lock'
else:
cmd += 'xdotool key ' + shlex.quote(key_str)
self.connection.run(cmd)
def mouse_cmd(self, pos):
x = (2 * pos.x() / self.mp.width() - 1)
y = (-2 * pos.y() / self.mp.height() + 1)
r = (x ** 2 + y ** 2) ** .5
theta = atan(y / x if x != 0 else copysign(float('inf'), y)) * 180 / pi
if r > CIRCLE_R3:
# Outside circle, do nothing
return
elif r > CIRCLE_R2:
# Move
dr = round((r - CIRCLE_R2) / (CIRCLE_R3 - CIRCLE_R2) * 400)
dtheta = round(90 - theta) + (0 if x > 0 else 180)
cmd = 'mousemove_relative --polar ' + str(dtheta) + ' ' + str(dr)
elif r > CIRCLE_R1:
if abs(theta) < 60:
if x >= 0:
# Right click
cmd = 'click 3'
else:
# Left click
cmd = 'click 1'
else:
if y >= 0:
# Scroll up
cmd = 'click 4'
else:
# Scroll down
cmd = 'click 5'
else:
# Middle click
cmd = 'click 2'
display = str(self.ui.displayField.text())
cmd = 'DISPLAY=' + shlex.quote(display) + ' xdotool ' + cmd
self.connection.run(cmd)
self.ui.typingField.setPlainText('')
# IP = os.getenv("ip")
# PORT = os.getenv("port")
# USER = os.getenv("BACKEND_AUTH_USR")
# PASS = os.getenv("BACKEND_AUTH_PSW")
# CONN = Connection(
# "{username}@{ip}:{port}".format(
# username=USER,
# ip=IP,
# port=PORT,
# ),
# connect_kwargs={"password": PASS},
# )
PEM_FILE = 'backend.pem'
DEV_SERVERS = SERVERS['dev']
for dev_server in DEV_SERVERS:
CONN = Connection(inline_ssh_env=PEM_FILE, host=dev_server['host'], user=dev_server['user'], \
connect_kwargs={"key_filename": PEM_FILE})
CONN.open()
print(CONN.is_connected)
with CONN.cd("/var/www/html"):
CONN.run("ls -a")
CONN.close()
def test_update_reg_clients_hosts(self, test_vars):
"""
Updates /etc/hosts on the STAF clients so they can contact the STAF
server.
"""
log = logging.getLogger("test_update_reg_clients_hosts")
atd = test_vars["atd_obj"]
commands = """
cp /etc/hosts .
echo ' ' >> hosts
echo '# STAF server IP' >> hosts
echo '{0} staf' >> hosts
sudo mv hosts /etc/hosts
echo '#!/bin/bash' > ~/hostdb_entries.sh
chmod 755 ~/hostdb_entries.sh
echo "cd ~/Avere-sv" >> ~/hostdb_entries.sh
echo "source /usr/sv/env/bin/activate" >> ~/hostdb_entries.sh
echo "export PYTHONPATH=~/Avere-sv:~/Avere-sv/averesv:$PYTHONPATH:$PATH" >> ~/hostdb_entries.sh
echo "averesv/hostdb.py -a vfxt -m {1} -p '{2}'" >> ~/hostdb_entries.sh
""".format(test_vars["staf_server_priv_ip"],
test_vars["cluster_mgmt_ip"],
os.environ["AVERE_ADMIN_PW"]).split("\n")
# Add hostdb entry calls for each regression client.
for i, staf_client_ip in enumerate(test_vars["staf_client_priv_ips"]):
commands.append(
"echo 'averesv/hostdb.py -L regclient{0} -m {1}' >> ~/hostdb_entries.sh"
.format(i, staf_client_ip))
# Get the storage account's access key and add that hostdb entry, too.
sa_key = atd.st_client.storage_accounts.list_keys(
atd.resource_group, test_vars["storage_account"]).keys[0].value
commands.append(
"echo 'averesv/hostdb.py -s {0}.blob.core.windows.net -m {0}.blob.core.windows.net -M az --cloudCreds \"{0}::{1}\"' >> ~/hostdb_entries.sh"
.format(test_vars["storage_account"], sa_key))
last_error = None
for staf_client_ip in test_vars["staf_client_priv_ips"]:
for port_attempt in range(1, 11):
tunnel_local_port = get_unused_local_port()
with Connection(test_vars["public_ip"],
user=test_vars["controller_user"],
connect_kwargs={
"key_filename": test_vars["ssh_priv_key"],
}).forward_local(local_port=tunnel_local_port,
remote_port=22,
remote_host=staf_client_ip):
node_c = Connection("127.0.0.1",
user=test_vars["controller_user"],
port=tunnel_local_port,
connect_kwargs={
"key_filename":
test_vars["ssh_priv_key"],
})
try:
node_c.open()
# If port_attempt > 1, last_error had the exception
# from the last iteration. Clear it.
last_error = None
except NoValidConnectionsError as ex:
last_error = ex
exp_err = "Unable to connect to port {} on 127.0.0.1".format(
tunnel_local_port)
if exp_err not in str(ex):
raise
else:
log.warning("{0} (attempt #{1}, retrying)".format(
exp_err, str(port_attempt)))
continue # iterate
run_ssh_commands(node_c.client, commands)
# Copy SSH keys to the client.
scp_cli = SCPClient(node_c.transport)
scp_cli.put(test_vars["ssh_priv_key"], "~/.ssh/id_rsa")
scp_cli.put(test_vars["ssh_pub_key"], "~/.ssh/id_rsa.pub")
scp_cli.close()
log.debug("Connection to {} closed".format(staf_client_ip))
break # no need to iterate again
if last_error:
log.error(
"See previous error(s) above. Raising last exception.")
raise last_error
def open_method_generates_real_connection(self):
c = Connection("localhost")
c.open()
assert c.client.get_transport().active is True
assert c.is_connected is True
return c
class RemoteRunner:
"""
Starts Jupyter lab on a remote resource and port forwards session to
local machine.
Returns
-------
RemoteRunner
An object that is responsible for connecting to remote host and launching jupyter lab.
Raises
------
SystemExit
When the specified local port is not available.
"""
host: str
port: int = 8888
conda_env: str = None
notebook_dir: str = None
port_forwarding: bool = True
launch_command: str = None
identity: str = None
shell: str = '/usr/bin/env bash'
def __post_init__(self):
if self.port_forwarding and not is_port_available(self.port):
raise SystemExit((
f'''Specified port={self.port} is already in use on your local machine. Try a different port'''
))
connect_kwargs = {}
if self.identity:
connect_kwargs['key_filename'] = [self.identity]
self.session = Connection(self.host,
connect_kwargs=connect_kwargs,
forward_agent=True)
try:
self.session.open()
except paramiko.ssh_exception.BadAuthenticationType:
loc_transport = self.session.client.get_transport()
loc_transport.auth_interactive_dumb(self.session.user,
_authentication_handler)
self.session.transport = loc_transport
def dir_exists(self, directory):
"""
Checks if a given directory exists on remote host.
"""
message = "couldn't find the directory"
cmd = f'''if [[ ! -d "{directory}" ]] ; then echo "{message}"; fi'''
out = self.session.run(cmd, hide='out').stdout.strip()
return message not in out
def setup_port_forwarding(self):
"""
Sets up SSH port forwarding
"""
print('**********************************')
print('*** Setting up port forwarding ***')
print('**********************************\n\n')
local_port = int(self.port)
remote_port = int(self.parsed_result['port'])
with self.session.forward_local(
local_port,
remote_port=remote_port,
remote_host=self.parsed_result['hostname'],
):
time.sleep(
3
) # don't want open_browser to run before the forwarding is actually working
open_browser(port=local_port, token=self.parsed_result['token'])
self.session.run(f'tail -f {self.log_file}', pty=True)
def close(self):
self.session.close()
def start(self):
"""
Launches Jupyter Lab on remote host, sets up ssh tunnel and opens browser on local machine.
"""
# jupyter lab will pipe output to logfile, which should not exist prior to running
# Logfile will be in $TMPDIR if defined on the remote machine, otherwise in $HOME
try:
if self.dir_exists('$TMPDIR'):
self.log_dir = '$TMPDIR'
else:
self.log_dir = '$HOME'
self.log_dir = f'{self.log_dir}/.jupyter_forward'
kwargs = dict(pty=True, shell=self.shell)
self.session.run(f'mkdir -p {self.log_dir}', **kwargs)
timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%S')
self.log_file = f'{self.log_dir}/log.{timestamp}'
self.session.run(f'touch {self.log_file}', **kwargs)
command = 'jupyter lab --no-browser'
if self.launch_command:
command = f'{command} --ip=\$(hostname)'
else:
command = f'{command} --ip=`hostname`'
if self.notebook_dir:
command = f'{command} --notebook-dir={self.notebook_dir}'
command = f'{command} > {self.log_file} 2>&1'
if self.conda_env:
command = f'conda activate {self.conda_env} && {command}'
if self.launch_command:
script_file = f'{self.log_dir}/batch-script.{timestamp}'
cmd = f"""echo "#!{self.shell}\n\n{command}" > {script_file}"""
self.session.run(cmd, **kwargs, echo=True)
self.session.run(f'chmod +x {script_file}', **kwargs)
command = f'{self.launch_command} {script_file}'
self.session.run(command, asynchronous=True, **kwargs, echo=True)
# wait for logfile to contain access info, then write it to screen
condition = True
stdout = None
pattern = 'is running at:'
while condition:
try:
result = self.session.run(f'cat {self.log_file}', **kwargs)
if pattern in result.stdout:
condition = False
stdout = result.stdout
except invoke.exceptions.UnexpectedExit:
print(
f'Trying to access {self.log_file} on {self.session.host} again...'
)
pass
self.parsed_result = parse_stdout(stdout)
if self.port_forwarding:
self.setup_port_forwarding()
else:
open_browser(url=self.parsed_result['url'])
self.session.run(f'tail -f {self.log_file}', **kwargs)
finally:
self.close()
print(
'\n***********************************************************'
)
print(
'*** Terminated the network connection to the remote end ***')
print(
'***********************************************************\n'
)
class r_sync(object):
def __init__(self, _cfg_file):
super().__init__()
self.cfg_file_ = _cfg_file
self.load_cfg()
def work(self):
self.open_conn()
self.check_sync()
self.close_conn()
pass
def load_cfg(self):
self.j_obj_ = None
with open(self.cfg_file_, 'r') as jsonFile:
self.j_obj_ = json.load(jsonFile)
self.r_ip_ = self.j_obj_["r_ip"]
self.r_usr_ = self.j_obj_["r_usr"]
self.r_path_ = self.j_obj_["r_path"]
self.keyfile_ = self.j_obj_["keyfile"]
self.l_path_ = self.j_obj_["l_path"]
def open_conn(self):
# 链接远程服务器的配置
self.conn_ = Connection(host=self.r_ip_,
user=self.r_usr_,
connect_kwargs={
"key_filename": self.keyfile_,
})
self.conn_.open()
logging.info(self.conn_.is_connected)
def check_sync(self):
old_cwd = os.getcwd()
os.chdir(self.l_path_)
self.recursion_path("./")
os.chdir(old_cwd)
def recursion_path(self, _path):
logging.info("dir: {0}".format(_path))
for root, dirs, files in os.walk(_path):
for dn in dirs:
sub_d = self.win2linux(str(_path) + "/" + str(dn))
if os.path.exists(sub_d) and os.listdir(sub_d):
self.recursion_path(sub_d)
for fn in files:
fn = self.win2linux(str(_path) + "/" + str(fn))
if os.path.exists(fn):
self.check_single_file(fn)
def check_single_file(self, _filename):
lmd5 = self.get_l_md5(_filename)
rmd5 = self.get_r_md5(_filename)
logging.info("local file: {0}, \nlmd5: {1}".format(_filename, lmd5))
logging.info("rmd5: {0}".format(rmd5))
if lmd5 != rmd5:
self.put_file(_filename)
def close_conn(self):
self.conn_.close()
def win2linux(self, _path):
_path = _path.replace("\\", "/")
_path = _path.replace("/./", "/")
_path = _path.replace("//", "/")
return _path
def get_r_md5(self, _name):
r_p = self.win2linux("{0}/{1}".format(self.r_path_, _name))
cmd = "mkdir -p {0}".format(os.path.dirname(r_p))
self.conn_.run(cmd)
cmd = "md5sum {0}".format(r_p) + "| awk '{print $1}'"
retstr = self.conn_.run(cmd)
return retstr.stdout[:-1]
def get_l_md5(self, _name):
if not os.path.isfile(_name):
return None
myhash = hashlib.md5()
f = open(_name, 'rb')
while True:
b = f.read(8096)
if not b:
break
myhash.update(b)
f.close()
return myhash.hexdigest()
def put_file(self, _name):
self.conn_.put(_name,
self.win2linux("{0}/{1}".format(self.r_path_, _name)))
from fabric import Connection
import os
from dotenv import load_dotenv
load_dotenv()
ip = os.getenv("ip")
port = os.getenv("port")
username = os.getenv("BACKEND_AUTH_USR")
password = os.getenv("BACKEND_AUTH_PSW")
conn = Connection(
"{username}@{ip}:{port}".format(
username=username,
ip=ip,
port=port,
),
connect_kwargs={"password": password},
)
conn.open()
print(conn.is_connected)
with conn.cd("/var/www/html"):
conn.run("ls -a")
conn.close()
class RemoteExecuter:
host = None
hostUrl = None
connectArgs = None
conn = None
sshConf = None
def __init__(self, host, user, password):
self.host = host
self.user = user
self.password = password
self.sshConf = sshConfig(host, user, password)
self.hostUrl = "{0}@{1}".format(self.sshConf.getUser(),
self.sshConf.getHost())
self.connectArgs = {"password": self.sshConf.getPass()}
self.conn = Connection(host=self.hostUrl,
port=self.sshConf.getPort(),
connect_kwargs=self.connectArgs)
def __init__(self, host, sshConf):
self.host = host
self.sshConf = sshConf
app.logger.info(
"connection info: {0}@{1} identified by {2}. passwordless {3}".
format(self.sshConf.getUser(), self.sshConf.getHost(),
self.sshConf.getPass(), self.sshConf.isPasswordless()))
self.hostUrl = "{0}@{1}".format(self.sshConf.getUser(),
self.sshConf.getHost())
if self.sshConf.isPasswordless() == True:
self.connectArgs = {
"key_filename": self.sshConf.getPassFile(),
"look_for_keys": False,
"allow_agent": False
}
else:
self.connectArgs = {"password": self.sshConf.getPass()}
self.conn = Connection(host=self.hostUrl,
port=self.sshConf.getPort(),
connect_kwargs=self.connectArgs)
def refreshConn(self):
self.conn.close()
self.conn.open()
def executeRemoteCommand(self, cmd):
try:
# cmd="sudo bash -c \"" + cmd + "\""
app.logger.info("executing remote command: {0}".format(cmd))
self.refreshConn()
result = self.conn.run(cmd, hide=True)
# result=self.conn.sudo(cmd,user="******",hide=True)
#result = self.conn.sudo(cmd, hide=True, shell=False)
return ReturnValue(result.exited, result.stderr, result.stdout)
except (UnexpectedExit, Exit, ParseError) as e:
if isinstance(e, ParseError):
app.logger.error(
"Remote Executor:: Parser failed with error:" + str(e))
return ReturnValue(1, e.stderr, "")
if isinstance(e, Exit) and e.message:
app.logger.error("Remote Executor:: Exited with error: " +
str(e))
return ReturnValue(e.code, e.stderr, "")
if isinstance(e, UnexpectedExit) and e.result.hide:
app.logger.error(
"Remote Executor:: Unexpected exit with error: " + str(e))
return ReturnValue(1, "", "")
else:
app.logger.error("Remote Executor:: Exception : " +
str(Exception) + " Err: " + str(e))
return ReturnValue(e.result.exited, e.result.stderr.strip(),
e.result.stdout.strip())
def copyFileToRemote(self, sourcePath, destinationPath):
try:
#conn = Connection(host=self.hostUrl, connect_kwargs=self.connectArgs)
app.logger.info("copying file to remote: {0}->{1}@{2}".format(
sourcePath, self.host, destinationPath))
self.refreshConn()
result = self.conn.put(sourcePath,
destinationPath,
preserve_mode=True)
#put() method doesnt return a fabric.runners.Result object like run() method.
#it returns a fabric.transfer.Result which has no return info.
#on failure it raises exception (oserror), no exception means it succeeded.
return ReturnValue(0, "", "")
except (UnexpectedExit, Exit, ParseError) as e:
if isinstance(e, ParseError):
app.logger.error(
"Remote Executor:: Parser failed with error:" + e)
return ReturnValue(1, e.stderr, "")
if isinstance(e, Exit) and e.message:
app.logger.error("Remote Executor:: Exited with error: " + e)
return ReturnValue(e.code, e.stderr, "")
if isinstance(e, UnexpectedExit) and e.result.hide:
return ReturnValue(1, "", "")
else:
app.logger.error("Remote Executor:: Exception : " +
str(Exception) + " Err: " + str(e))
return ReturnValue(e.result.exited, e.result.stderr.strip(),
e.result.stdout.strip())
def opsHandler(self, opList):
# execute operations remotely one by one
for cmd in opList:
try:
if cmd["operId"] == RemoteOperations.CMD:
self.executeRemoteCommand(cmd["args"][0])
elif cmd["operId"] == RemoteOperations.COPY:
self.copyFileToRemote(cmd["args"][0], cmd["args"][1])
else:
app.logger.error(
"remote operation {0} is not supported".format(
cmd["operId"]))
raise Exception(
"remote operation {0} is not supported".format(
cmd["operId"]))
except Exception as ex:
app.logger.error(
"operation {0} failed with exception {1}".format(
str(cmd), str(ex)))
return True
def get_connection(self, ssh_host, ssh_gateway=None):
"""
Establish ssh connection to remote host(s).
Arguments:
ssh_host {ConnectionParts}
ssh_gateway {ConnectionParts}
Returns:
Connection -- an instance of a fabric Connection object that represents a valid ssh
connection to the remote host.
Note:
The current implementation assumes that the username, port, and password are the
same for the ssh host and the gateway.
"""
password = ssh_host.password
if not password:
prompt = '{}@{} password: '******'Exiting.')
sys.exit(1)
# TODO: Abstract the gateway connection to a more generalized "connection" method so we
# can more specifically pinpoint (in the event of a connection error) whether the error
# occurred while trying to connect to the host or the gateway.
try:
# establish the ssh connection
connection = Connection(
host=ssh_host.hostname,
port=ssh_host.port,
user=ssh_host.username,
gateway=Connection(host=ssh_gateway.hostname,
user=ssh_gateway.username,
port=ssh_gateway.port,
connect_kwargs={'password': password})
if ssh_gateway is not None else None,
connect_kwargs={'password': password})
# open the ssh connection to test the connection
connection.open()
_LOGGER.info('Connected to host: %s.', ssh_host.hostname)
return connection
# FIXME: Allow for authentication error a couple times before exiting due to wrong password
# TODO: Better handle different kinds of exceptions.
except Exception as err:
_LOGGER.error('Failed to connect to host "%s". %s',
ssh_host.hostname, err)
sys.exit(1)
def test_artifacts_collect(self, averecmd_params, scp_con,
test_vars): # noqa: F811, E501
"""
Collect test artifacts (node logs, rolling trace) from each node.
Artifacts are stored to local directories.
"""
log = logging.getLogger("test_collect_artifacts")
artifacts_dir = "vfxt_artifacts_" + test_vars["atd_obj"].deploy_id
os.makedirs(artifacts_dir, exist_ok=True)
log.debug("Copying logs from controller to {}".format(artifacts_dir))
for lf in [
"vfxt.log", "enablecloudtrace.log",
"create_cluster_command.log"
]:
scp_con.get("~/" + lf, artifacts_dir)
log.debug("Copying SSH keys to the controller")
scp_con.put(test_vars["ssh_priv_key"], "~/.ssh/.")
scp_con.put(test_vars["ssh_pub_key"], "~/.ssh/.")
nodes = run_averecmd(**averecmd_params, method="node.list")
log.debug("Nodes found: {}".format(nodes))
last_error = None
for node in nodes:
node_dir = artifacts_dir + "/" + node
node_dir_log = node_dir + "/log"
node_dir_trace = node_dir + "/trace"
log.debug("node_dir_log = {}, node_dir_trace = {}".format(
node_dir_log, node_dir_trace))
# make local directories to store downloaded artifacts
os.makedirs(node_dir_trace, exist_ok=True)
os.makedirs(node_dir_log, exist_ok=True)
# get this node's primary cluster IP address
node_ip = run_averecmd(**averecmd_params,
method="node.get",
args=node)[node]["primaryClusterIP"]["IP"]
log.debug("Tunneling to node {} using IP {}".format(node, node_ip))
# get_unused_local_port actually uses the port to know it's
# available before making it available again and returning the
# port number. Rarely, there is a race where the open() call
# below fails because the port is not yet fully available
# again. In those cases, try getting a new port.
for port_attempt in range(1, 11):
tunnel_local_port = get_unused_local_port()
with Connection(test_vars["public_ip"],
user=test_vars["controller_user"],
connect_kwargs={
"key_filename": test_vars["ssh_priv_key"],
}).forward_local(local_port=tunnel_local_port,
remote_port=22,
remote_host=node_ip):
node_c = Connection("127.0.0.1",
user="******",
port=tunnel_local_port,
connect_kwargs={
"password":
os.environ["AVERE_ADMIN_PW"]
})
try:
node_c.open()
# If port_attempt > 1, last_error had the exception
# from the last iteration. Clear it.
last_error = None
except NoValidConnectionsError as ex:
last_error = ex
exp_err = "Unable to connect to port {} on 127.0.0.1".format(
tunnel_local_port)
if exp_err not in str(ex):
raise
else:
log.warning("{0} (attempt #{1}, retrying)".format(
exp_err, str(port_attempt)))
continue # iterate
scp_client = SCPClient(node_c.transport)
try:
# Calls below catch exceptions and report them to the
# error log, but then continue. This is because a
# failure to collect artifacts on one node should not
# prevent collection from other nodes. After collection
# has completed, the last exception will be raised.
# list of files and directories to download
to_collect = [
"/var/log/messages",
"/var/log/xmlrpc.log",
# assumes rolling trace was enabled during deploy
"/support/trace/rolling",
# TODO: 2019-0219: turned off for now
# "/support/gsi",
# "/support/cores",
]
for tc in to_collect:
log.debug("SCP'ing {} from node {} to {}".format(
tc, node, node_dir_log))
try:
scp_client.get(tc,
node_dir_log,
recursive=True)
except Exception as ex:
log.error("({}) Exception caught: {}".format(
node, ex))
last_error = ex
finally:
scp_client.close()
log.debug("Connections to node {} closed".format(node))
break # no need to iterate again
if last_error:
log.error("See previous error(s) above. Raising last exception.")
raise last_error
def try_login(current_host, default_user, user_pass, itadm_password,
root_password, fallback_itadm_password):
try:
conn = Connection(host=default_user + "@" + current_host,
connect_timeout=1,
connect_kwargs={"password": user_pass})
conn.open()
return conn
except Exception as e:
try:
conn = Connection(host="itadm@" + current_host,
connect_timeout=1,
connect_kwargs={"password": itadm_password})
conn.open()
return conn
except Exception as e:
try:
conn = Connection(
host="itadm@" + current_host,
connect_timeout=1,
connect_kwargs={"password": fallback_itadm_password})
conn.open()
return conn
except Exception as e:
try:
conn = Connection(
host="root@" + current_host,
connect_timeout=1,
connect_kwargs={"password": root_password})
conn.open()
return conn
except Exception as e:
try:
conn = Connection(
host="itadm@" + current_host,
connect_timeout=1,
port=8567,
connect_kwargs={"password": itadm_password})
conn.open()
return conn
except Exception as e:
try:
conn = Connection(host="itadm@" + current_host,
connect_timeout=1,
port=8567,
connect_kwargs={
"password":
fallback_itadm_password
})
conn.open()
return conn
except Exception as e:
try:
conn = Connection(
host="root@" + current_host,
connect_timeout=1,
port=8567,
connect_kwargs={"password": root_password})
conn.open()
return conn
except Exception as e:
return None
def create():
"""
Create a new Digital Ocean droplet and then deploy the php server code.
For example: Typing "fab create BobServer nyc1 s-1vcpu-1gb" on the command
line will create a new Digital Ocean Droplet named "BobServer" in
the NYC1 region, with a size of 512mb. It will automatically pull
your ssh keys and store the public keys on the new server so that
you can access it as root right away.
"""
manager = digitalocean.Manager(token=DIGITAL_OCEAN_TOKEN)
current_names = _get_current_droplets(manager)
username, name, region, size_slug = _get_parameters(current_names)
ssh_keys = _create_ssh_keys(manager)
droplet = digitalocean.Droplet(
token=DIGITAL_OCEAN_TOKEN,
name=name,
region=region,
image='ubuntu-16-04-x64',
size_slug=size_slug,
ssh_keys=ssh_keys,
backups=False,
)
print('Creating droplet...')
droplet.create()
print('Verifying...')
actions = droplet.get_actions()
for action in actions:
status = ''
while status != 'completed':
action.load()
status = action.status
# Once it shows completed, droplet is up and running
print(action.status)
droplet.load()
if not droplet.id:
raise ValueError('Droplet creation failed. No ID.')
info = f'{droplet.name} ip_address: {droplet.ip_address}'
print(f'Droplet created: {info}')
with open('droplet_data.txt', 'w') as file:
file.write(info + '\n')
if not droplet.ip_address:
raise ValueError('Droplet creation failed. No ip address registered.')
key_path = os.path.join(SSH_PATH, DIGITAL_OCEAN_PRIVATE_KEY)
if DIGITAL_OCEAN_KEY_PASSPHRASE:
connect_kwargs = {'key_filename': key_path, 'passphrase': DIGITAL_OCEAN_KEY_PASSPHRASE}
else:
connect_kwargs = {'key_filename': key_path}
delay = 10
print(f"Let's wait {delay} seconds to make sure it's finished starting up.")
time.sleep(delay)
print("Okay, let's try to connect...")
connection = Connection(
host=droplet.ip_address,
user='******',
connect_kwargs=connect_kwargs
)
try:
connection.open()
except (TimeoutError, NoValidConnectionsError):
connection.close()
print(f"Connection failed. Let's wait {delay} more seconds and try again...")
time.sleep(delay)
connection = Connection(
host=droplet.ip_address,
user='******',
connect_kwargs=connect_kwargs
)
try:
connection.open()
except (TimeoutError, NoValidConnectionsError):
print("Connection failed. Exiting...")
return None, None, None
username, password = new_user(username, connection)
with open('droplet_data.txt', 'a') as file:
file.writelines([
f'username: {username} group: admin password: {password}',
'sudo privileges granted.',
])
return droplet.ip_address, username, password
class FabricRunner(object):
def __init__(self,
logger=None,
host=None,
user=None,
key=None,
port=None,
password=None,
validate_connection=True,
fabric_env=None,
tmpdir=None):
# logger
self.logger = logger or setup_logger('fabric_runner')
# silence paramiko
logging.getLogger('paramiko.transport').setLevel(logging.WARNING)
# connection details
self.port = port or DEFAULT_REMOTE_EXECUTION_PORT
self.password = password
self.user = user
self.host = host
self.key = key
self.tmpdir = tmpdir
# fabric environment
self.env = self._set_env()
self.env.update(fabric_env or {})
self._connection = None
self._validate_ssh_config()
if validate_connection:
self.validate_connection()
def _validate_ssh_config(self):
if not self.host:
raise exceptions.AgentInstallerConfigurationError('Missing host')
if not self.user:
raise exceptions.AgentInstallerConfigurationError('Missing user')
if not is_kerberos_env() and not self.password and not self.key:
raise exceptions.AgentInstallerConfigurationError(
'Must specify either key or password')
def _load_private_key(self, key_contents):
"""Load the private key and return a paramiko PKey subclass.
:param key_contents: the contents of a keyfile, as a string starting
with "---BEGIN"
:return: A paramiko PKey subclass - RSA, ECDSA or Ed25519
"""
for cls in (RSAKey, ECDSAKey, Ed25519Key):
try:
return cls.from_private_key(StringIO(key_contents))
except SSHException:
continue
raise exceptions.AgentInstallerConfigurationError(
'Could not load the private key as an '
'RSA, ECDSA, or Ed25519 key')
def _set_env(self):
env = {
'host': self.host,
'port': self.port,
'user': self.user,
'connect_kwargs': {}
}
if self.key:
if self.key.startswith(PRIVATE_KEY_PREFIX):
env['connect_kwargs']['pkey'] = \
self._load_private_key(self.key)
else:
env['connect_kwargs']['key_filename'] = self.key
if self.password:
env['connect_kwargs']['password'] = self.password
if is_kerberos_env():
# For GSSAPI, the fabric env just needs to have
# gss_auth and gss_kex set to True
env['gss_auth'] = True
env['gss_kex'] = True
env.update(COMMON_ENV)
return env
def validate_connection(self):
self.logger.debug('Validating SSH connection')
self.ping()
self.logger.debug('SSH connection is ready')
def _ensure_connection(self):
if self._connection is None:
self._connection = Connection(**self.env)
try:
self._connection.open()
except Exception as e:
_, _, tb = sys.exc_info()
reraise(FabricCommandExecutionError,
FabricCommandExecutionError(str(e)), tb)
def run(self, command, execution_env=None, **attributes):
"""
Execute a command.
:param command: The command to execute.
:param execution_env: environment variables to be applied before
running the command
:param quiet: run the command silently
:param attributes: custom attributes passed directly to
fabric's run command
:return: a response object containing information
about the execution
:rtype: FabricCommandExecutionResponse
"""
if execution_env is None:
execution_env = {}
self._ensure_connection()
attributes.setdefault('hide', self.logger.isEnabledFor(logging.DEBUG))
attributes.setdefault('warn', True)
r = self._connection.run(command, **attributes)
if r.return_code != 0:
raise FabricCommandExecutionException(command=command,
error=r.stderr,
output=r.stdout,
code=r.return_code)
return FabricCommandExecutionResponse(command=command,
std_out=r.stdout,
std_err=None,
return_code=r.return_code)
def sudo(self, command, **attributes):
"""
Execute a command under sudo.
:param command: The command to execute.
:param attributes: custom attributes passed directly to
fabric's run command
:return: a response object containing information
about the execution
:rtype: FabricCommandExecutionResponse
"""
return self.run('sudo {0}'.format(command), **attributes)
def run_script(self, script):
"""
Execute a script.
:param script: The path to the script to execute.
:return: a response object containing information
about the execution
:rtype: FabricCommandExecutionResponse
:raise: FabricCommandExecutionException
"""
remote_path = self.put_file(script)
try:
self.sudo('chmod +x {0}'.format(remote_path))
result = self.sudo(remote_path)
finally:
# The script is pushed to a remote directory created with mkdtemp.
# Hence, to cleanup the whole directory has to be removed.
self.delete(os.path.dirname(remote_path))
return result
def put_file(self, src, dst=None, sudo=False, **attributes):
"""
Copies a file from the src path to the dst path.
:param src: Path to a local file.
:param dst: The remote path the file will copied to.
:param sudo: indicates that this operation
will require sudo permissions
:param attributes: custom attributes passed directly to
fabric's run command
:return: the destination path
"""
if dst:
self.verify_dir_exists(os.path.dirname(dst))
else:
basename = os.path.basename(src)
tempdir = self.mkdtemp()
dst = os.path.join(tempdir, basename)
self._ensure_connection()
if dst is None:
dst = os.path.basename(src)
target_path = dst
if sudo:
dst = os.path.basename(dst)
self._connection.put(src, dst)
if sudo:
self.sudo('sudo mv {0} {1}'.format(dst, target_path))
return target_path
def ping(self, **attributes):
"""
Tests that the connection is working.
:param attributes: custom attributes passed directly to
fabric's run command
:return: a response object containing information
about the execution
:rtype: FabricCommandExecutionResponse
"""
return self.run('echo', **attributes)
def mktemp(self, create=True, directory=False, **attributes):
"""
Creates a temporary path.
:param create: actually create the file or just construct the path
:param directory: path should be a directory or not.
:param attributes: custom attributes passed directly to
fabric's run command
:return: the temporary path
"""
flags = []
if not create:
flags.append('-u')
if directory:
flags.append('-d')
if self.tmpdir is not None:
flags.append('-p "{0}"'.format(self.tmpdir))
return self.run('mktemp {0}'.format(' '.join(flags)),
**attributes).std_out.rstrip()
def mkdtemp(self, create=True, **attributes):
"""
Creates a temporary directory path.
:param create: actually create the file or just construct the path
:param attributes: custom attributes passed directly to
fabric's run command
:return: the temporary path
"""
return self.mktemp(create=create, directory=True, **attributes)
def home_dir(self, username):
"""
Retrieve the path of the user's home directory.
:param username: the username
:return: path to the home directory
"""
return self.python(
imports_line='import pwd',
command='pwd.getpwnam(\'{0}\').pw_dir'.format(username))
def verify_dir_exists(self, dirname):
self.run('mkdir -p {0}'.format(dirname))
def python(self, imports_line, command, **attributes):
"""
Run a python command and return the output.
To overcome the situation where additional info is printed
to stdout when a command execution occurs, a string is
appended to the output. This will then search for the string
and the following closing brackets to retrieve the original output.
:param imports_line: The imports needed for the command.
:param command: The python command to run.
:param attributes: custom attributes passed directly to
fabric's run command
:return: the string representation of the return value of
the python command
"""
python_bin = '$(command which python ' \
'|| command which python3 ' \
'|| echo "python")'
start = '###CLOUDIFYCOMMANDOPEN'
end = 'CLOUDIFYCOMMANDCLOSE###'
stdout = self.run(
'{0} -c "import sys; {1}; '
'sys.stdout.write(\'{2}{3}{4}\\n\''
'.format({5}))"'.format(python_bin, imports_line, start, '{0}',
end, command), **attributes).std_out
result = stdout[stdout.find(start) - 1 + len(end):stdout.find(end)]
return result
def machine_distribution(self, **attributes):
"""
Retrieves the distribution information of the host.
:param attributes: custom attributes passed directly to
fabric's run command
:return: dictionary of the platform distribution as returned from
'platform.dist()'
"""
response = self.python(imports_line='import platform, json',
command='json.dumps(platform.dist())',
**attributes)
return api_utils.json_loads(response)
def delete(self, path):
self.run('rm -rf {0}'.format(path))
def close(self):
if self._connection is not None:
self._connection.close()
def open_method_generates_real_connection(self):
c = Connection('localhost')
c.open()
eq_(c.client.get_transport().active, True)
eq_(c.is_connected, True)
return c
class RemoteRunner:
"""
Starts Jupyter lab on a remote resource and port forwards session to
local machine.
Returns
-------
RemoteRunner
An object that is responsible for connecting to remote host and launching jupyter lab.
Raises
------
SystemExit
When the specified local port is not available.
"""
host: str
port: int = 8888
conda_env: str = None
notebook_dir: str = None
port_forwarding: bool = True
launch_command: str = None
identity: str = None
shell: str = '/usr/bin/env bash'
def __post_init__(self):
self.run_kwargs = dict(pty=True)
console.rule('[bold green]Authentication', characters='*')
if self.port_forwarding and not is_port_available(self.port):
console.log(
f'''[bold red]Specified port={self.port} is already in use on your local machine. Try a different port'''
)
sys.exit(1)
connect_kwargs = {}
if self.identity:
connect_kwargs['key_filename'] = [str(self.identity)]
self.session = Connection(self.host, connect_kwargs=connect_kwargs, forward_agent=True)
console.log(
f'[bold cyan]Authenticating user ({self.session.user}) from client ({socket.gethostname()}) to remote host ({self.session.host})'
)
# Try passwordless authentication
try:
self.session.open()
except (
paramiko.ssh_exception.BadAuthenticationType,
paramiko.ssh_exception.AuthenticationException,
):
pass
# Prompt for password and token (2FA)
if not self.session.is_connected:
for _ in range(2):
try:
loc_transport = self.session.client.get_transport()
try:
loc_transport.auth_interactive_dumb(
self.session.user, _authentication_handler
)
except paramiko.ssh_exception.BadAuthenticationType:
# It is not clear why auth_interactive_dumb fails in some cases, but
# in the examples we could generate auth_password was successful
loc_transport.auth_password(self.session.user, getpass.getpass())
self.session.transport = loc_transport
break
except Exception:
console.log('[bold red]:x: Failed to Authenticate your connection')
if not self.session.is_connected:
sys.exit(1)
console.log('[bold cyan]:white_check_mark: The client is authenticated successfully')
def _jupyter_info(self, command='sh -c "command -v jupyter"'):
console.rule('[bold green]Running jupyter sanity checks', characters='*')
out = self.session.run(command, warn=True, hide='out', **self.run_kwargs)
if out.failed:
console.log(f"[bold red]:x: Couldn't find jupyter executable with: '{command}'")
sys.exit(1)
console.log('[bold cyan]:white_check_mark: Found jupyter executable')
def envvar_exists(self, envvar):
message = 'variable is not defined'
cmd = f'''printenv {envvar} || echo "{message}"'''
out = self.session.run(cmd, hide='out', **self.run_kwargs).stdout.strip()
return message not in out
def dir_exists(self, directory):
"""
Checks if a given directory exists on remote host.
"""
message = "couldn't find the directory"
cmd = f'''cd {directory} || echo "{message}"'''
out = self.session.run(cmd, hide='out', **self.run_kwargs).stdout.strip()
return message not in out
def setup_port_forwarding(self):
"""
Sets up SSH port forwarding
"""
console.rule('[bold green]Setting up port forwarding', characters='*')
local_port = int(self.port)
remote_port = int(self.parsed_result['port'])
with self.session.forward_local(
local_port,
remote_port=remote_port,
remote_host=self.parsed_result['hostname'],
):
time.sleep(
3
) # don't want open_browser to run before the forwarding is actually working
open_browser(port=local_port, token=self.parsed_result['token'])
self.session.run(f'tail -f {self.log_file}', pty=True)
def close(self):
self.session.close()
def start(self):
"""
Launches Jupyter Lab on remote host, sets up ssh tunnel and opens browser on local machine.
"""
# jupyter lab will pipe output to logfile, which should not exist prior to running
# Logfile will be in $TMPDIR if defined on the remote machine, otherwise in $HOME
try:
check_jupyter_status = 'sh -c "command -v jupyter"'
if self.conda_env:
check_jupyter_status = (
f'conda activate {self.conda_env} && sh -c "command -v jupyter"'
)
self._jupyter_info(check_jupyter_status)
if self.envvar_exists('TMPDIR') and self.dir_exists('$TMPDIR'):
self.log_dir = '$TMPDIR'
elif self.envvar_exists('HOME') and self.dir_exists('$HOME'):
self.log_dir = '$HOME'
else:
message = (
'$TMPDIR/ is not a directory'
if self.envvar_exists('TMPDIR')
else '$TMPDIR is not defined'
)
console.log(f'[bold red]{message}')
message = (
'$HOME/ is not a directory'
if self.envvar_exists('HOME')
else '$HOME is not defined'
)
console.log(f'[bold red]{message}')
console.log('[bold red]Can not determine directory for log file')
sys.exit(1)
self.log_dir = f'{self.log_dir}/.jupyter_forward'
self.session.run(f'mkdir -p {self.log_dir}', **self.run_kwargs)
timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%S')
self.log_file = f'{self.log_dir}/log.{timestamp}'
self.session.run(f'touch {self.log_file}', **self.run_kwargs)
command = 'jupyter lab --no-browser'
if self.launch_command:
command = f'{command} --ip=\$(hostname)'
else:
command = f'{command} --ip=`hostname`'
if self.notebook_dir:
command = f'{command} --notebook-dir={self.notebook_dir}'
command = f'{command} >& {self.log_file}'
if self.conda_env:
command = f'conda activate {self.conda_env} && {command}'
if self.launch_command:
script_file = f'{self.log_dir}/batch-script.{timestamp}'
cmd = f"""echo "#!{self.shell}\n\n{command}" > {script_file}"""
self.session.run(cmd, **self.run_kwargs, echo=True)
self.session.run(f'chmod +x {script_file}', **self.run_kwargs)
command = f'{self.launch_command} {script_file}'
self.session.run(command, asynchronous=True, **self.run_kwargs, echo=False)
# wait for logfile to contain access info, then write it to screen
condition = True
stdout = None
pattern = 'is running at:'
with console.status(
f'[bold cyan]Parsing {self.log_file} log file on {self.session.host} for jupyter information',
spinner='weather',
):
while condition:
try:
result = self.session.run(
f'cat {self.log_file}', **self.run_kwargs, echo=False
)
if pattern in result.stdout:
condition = False
stdout = result.stdout
except invoke.exceptions.UnexpectedExit:
pass
self.parsed_result = parse_stdout(stdout)
if self.port_forwarding:
self.setup_port_forwarding()
else:
open_browser(url=self.parsed_result['url'])
self.session.run(f'tail -f {self.log_file}', **self.run_kwargs)
except Exception:
self.close()
finally:
console.rule(
'[bold red]Terminated the network 📡 connection to the remote end', characters='*'
)
class SSH(Resource):
# Generic properties of any Resource
name = attrib(default=attr.NOTHING)
# Configurable options for each "instance"
host = attrib(default=attr.NOTHING, doc="DNS or IP address of server")
port = attrib(doc="Port to connect to on remote host")
key_filename = attrib(
doc="Path to SSH private key file matched with AWS key name parameter")
user = attrib(doc="Username to use to log into remote environment")
id = attrib() # EC2 instance ID
type = attrib(default='ssh') # Resource type
# Current instance properties, to be set by us, not augmented by user
status = attrib()
_connection = attrib()
def connect(self, password=None):
"""Open a connection to the environment resource.
Parameters
----------
password : string
We don't allow users to pass in a password via the command line
but do allow tests to authenticate by passing a password as
a parameter to this method.
"""
self._connection = Connection(self.host,
user=self.user,
port=self.port,
connect_kwargs={
'key_filename': self.key_filename,
'password': password
})
if self.key_filename:
auth = self.key_filename
elif password is None:
auth = "SSH config"
else:
auth = "password"
lgr.debug(
"SSH connecting to %s@%s:%s, authenticating with %s",
self._connection.user,
self._connection.host,
self._connection.port, # Fabric defaults to 22.
auth)
try:
self._connection.open()
except AuthenticationException:
password = getpass.getpass()
self._connection = Connection(
self.host,
user=self.user,
port=self.port,
connect_kwargs={'password': password})
self._connection.open()
def create(self):
"""
Register the SSH connection to the niceman inventory registry.
Returns
-------
dict : config and state parameters to capture in the inventory file
"""
if not self.id:
self.id = str(uuid.uuid4())
self.status = 'N/A'
return {
'id': self.id,
'status': self.status,
'host': self.host,
'user': self.user,
'port': self.port,
'key_filename': self.key_filename,
}
def delete(self):
self._connection = None
return
def start(self):
# Not a SSH feature
raise NotImplementedError
def stop(self):
# Not a SSH feature
raise NotImplementedError
def get_session(self, pty=False, shared=None):
"""
Log into remote environment and get the command line
"""
if not self._connection:
self.connect()
return (PTYSSHSession if pty else SSHSession)(
connection=self._connection)
