def _CreateFabricContext(ssh_config):
"""Create a fabric context."""
# fabric has problems to understand complex ssh config.
# Point to a user config
fab_config = fabric.Config(system_ssh_path='~/.ssh/config')
connect_kwargs = {
'config': fab_config
}
if ssh_config.ssh_key or ssh_config.password:
connect_kwargs['connect_kwargs'] = {}
if ssh_config.ssh_key:
logger.debug(
'Use ssh key %s ssh %s@%s.', ssh_config.ssh_key,
ssh_config.user, ssh_config.hostname)
connect_kwargs['connect_kwargs']['key_filename'] = ssh_config.ssh_key
if ssh_config.password:
logger.debug('Use password to ssh %s@%s.',
ssh_config.user, ssh_config.hostname)
connect_kwargs['connect_kwargs']['password'] = ssh_config.password
connect_kwargs['user'] = ssh_config.user
try:
wrapped_context = fabric.Connection(
host=ssh_config.hostname, **connect_kwargs)
wrapped_context.open()
return wrapped_context
except (paramiko.AuthenticationException, paramiko.SSHException) as e:
logger.debug(e)
raise AuthenticationError(e)
def init_connection(self,
host,
user=None,
port=None,
gateway=None,
ssh_config_path=None,
args=None):
"""
init_connection provides implementation of SSH plugin's init_connection method:
self.targets[name].init_connection(host, user, port, gateway, ssh_config_path, args)
"""
try:
if self.connection and self.connection.is_connected:
log.debug("Closing connection to {}".format(
self.connection.host))
self.connection.close()
if ssh_config_path:
ssh_config_path = expand_path(ssh_config_path)
config = fabric.Config(runtime_ssh_path=ssh_config_path)
self.connection = fabric.Connection(host,
user=user,
port=port,
gateway=gateway,
config=config,
connect_kwargs=args)
log.debug("Opening connection to {}...".format(
self.connection.host))
self.connection.open()
return True
except Exception as exception:
log.error("Remote connection failed: {}".format(exception))
raise CtfTestError("Error in init_connection") from exception
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._adb_params = {
"port": 22350,
"ip": "127.0.0.1",
"stdout": "adb-server.out",
"stderr": "adb-server.err",
"key": ".adb_key",
}
# the local connection
self._invoke_config = invoke.Config(
{"run": {
"warn": True,
"hide": True
}})
self._fabric_config = fabric.Config(
{"run": {
"hide": True,
"warn": True
}})
self._connection = invoke.Context(self._invoke_config)
self._spawned = False
self._local_adb_version = None
self._remote_adb_version = None
def connect(self) -> fabric.Connection:
if self.__c is None:
pwd = self.__passwordProvider(self.__hostName, self.__userName)
config = fabric.Config(overrides={'sudo': {'password': pwd}})
self.__c = fabric.Connection(self.__hostName, self.__userName, self.__port, config=config, connect_kwargs={"password": pwd})
r = self.__c.run(
self._encodeSSHCmdLine("/bin/echo", "foo \" \" bar"),
hide=True)
Assert.isTrue(self.__c.is_connected)
Assert.isEqual(r.exited, 0)
Assert.isEqual(r.stderr, "")
Assert.isEqual(r.stdout, "foo \" \" bar\n")
return self.__c
def sudo_remote(host, cmd):
try:
config = fabric.Config(
overrides={'sudo': {
'password': cockroach_config['password']
}})
conn = fabric.Connection(host, config=config)
conn.sudo(cmd)
return True
except invoke.exceptions.UnexpectedExit as e:
print("[ERROR] Remote run on host[{0}] cmd[{1}]".format(host, cmd))
print("[ERROR] Exception: " + str(e))
return False
except:
print("[ERROR] Remote run on host[{0}] cmd[{1}]".format(host, cmd))
print("[ERRPR] Some unknown error happend")
return False
def _purge_filesystem(
host: str, pid: str, location: dict, dryrun: bool, password: str
):
config = fabric.Config(overrides={"sudo": {"password": password}})
for retry in range(0, 3):
with fabric.Connection(host, config=config, connect_timeout=60) as c:
cmd = f"rm -rf {location}/{pid}"
if not dryrun:
logger.info(f"{cmd}")
r = c.sudo(f"{cmd}")
if r.ok:
logger.info(r.stdout)
break
else:
time.sleep(30)
else:
logger.info(f"DRYRUN: {cmd}")
break
def init(self, **kwargs):
self.remote_path = kwargs.pop("remote_path", None)
self.remote_path_owner = kwargs.pop("remote_path_owner", None)
self.remote_path_group = kwargs.pop("remote_path_group", None)
self.sudo = kwargs.pop("sudo", False)
self.sudo_password = kwargs.pop("sudo_password", None)
if not self.remote_path:
raise ValueError("'remote_path' is required")
if self.sudo is True:
kwargs.setdefault(
"config",
fabric.Config(overrides=dict(sudo=dict(password=self.sudo_password))),
)
self.connection = fabric.Connection(**kwargs)
self.tmp_dir = tempfile.TemporaryDirectory()
super(SCPOutputDriver, self).init(output_dir=self.tmp_dir.name)
def init_connection(self, host, user=None, port=None, gateway=None, ssh_config_path=None, args=None):
try:
if self.connection and self.connection.is_connected:
log.debug("Closing connection to {}".format(self.connection.host))
self.connection.close()
if ssh_config_path:
ssh_config_path = expand_path(ssh_config_path)
config = fabric.Config(runtime_ssh_path=ssh_config_path)
self.connection = fabric.Connection(host,
user=user,
port=port,
gateway=gateway,
config=config,
connect_kwargs=args)
log.debug("Opening connection to {}...".format(self.connection.host))
self.connection.open()
return True
except Exception as e:
log.error("Remote connection failed: {}".format(e))
return False
def qlaunch():
m_description = 'This program is used to submit jobs to a queueing system. ' \
'Details of the job and queue interaction are handled by the ' \
'mandatory queue adapter file parameter. The "rapidfire" option ' \
'can be used to maintain a certain number of jobs in the queue by ' \
'specifying the n_loops parameter to a large number. If n_loops is ' \
'set to 1 (default) the queue launcher will quit after submitting ' \
'the desired number of jobs. For more help on rapid fire options, ' \
'use qlauncher.py rapidfire -h'
parser = ArgumentParser(description=m_description)
subparsers = parser.add_subparsers(help='command', dest='command')
single_parser = subparsers.add_parser('singleshot', help='launch a single rocket to the queue')
rapid_parser = subparsers.add_parser('rapidfire', help='launch multiple rockets to the queue')
parser.add_argument("-rh", "--remote_host", nargs="*",
help="Remote host to exec qlaunch. Right now, "
"only supports running from a config dir.")
parser.add_argument("-rc", "--remote_config_dir",
nargs="+",
help="Remote config dir location(s). Defaults to "
"~/.fireworks. You can specify multiple "
"locations if you have multiple configurations "
"on the same cluster e.g., multiple queues or FireWorkers. "
"Note that this may have to come before the -ru"
"argument (or other single arg) options as "
"argparse may not be able to find "
"the find command while it consumes args.",
default=["~/.fireworks"])
parser.add_argument("-ru", "--remote_user",
help="Username to login to remote host.")
parser.add_argument("-rp", "--remote_password",
help="Password for remote host (if necessary). For "
"best operation, it is recommended that you do "
"passwordless ssh.")
parser.add_argument("-rsh", "--remote_shell",
help="Shell command to use on remote host for running submission.",
default='/bin/bash -l -c')
parser.add_argument("-rkf", "--remote_keyfile",
help="SSH keyfile for connecting to remote hosts",
type=str, default=None)
parser.add_argument("-rs", "--remote_setup",
help="Setup the remote config dir using files in "
"the directory specified by -c.",
action="store_true")
parser.add_argument("-d", "--daemon",
help="Daemon mode. Command is repeated every x "
"seconds. Defaults to 0, which means non-daemon "
"mode.",
type=int,
default=0)
parser.add_argument('--launch_dir', help='directory to launch the job / rapid-fire', default='.')
parser.add_argument('--logdir', help='path to a directory for logging', default=None)
parser.add_argument('--loglvl', help='level to print log messages', default='INFO')
parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true')
parser.add_argument('-r', '--reserve', help='reserve a fw', action='store_true')
parser.add_argument('-l', '--launchpad_file', help='path to launchpad file')
parser.add_argument('-w', '--fworker_file', help='path to fworker file')
parser.add_argument('-q', '--queueadapter_file', help='path to queueadapter file')
parser.add_argument('-c', '--config_dir',
help='path to a directory containing the config file (used if -l, -w, -q unspecified)',
default=CONFIG_FILE_DIR)
parser.add_argument('-fm', '--fill_mode', help='launch queue submissions even when there is nothing to run',
action='store_true')
rapid_parser.add_argument('-m', '--maxjobs_queue',
help='maximum jobs to keep in queue for this user. 0 for no limit', default=0,
type=int)
rapid_parser.add_argument('-b', '--maxjobs_block',
help='maximum jobs to put in a block',
default=500, type=int)
rapid_parser.add_argument('--nlaunches',
help='num_launches (int or "infinite"; default 0 is all jobs in DB)',
default=0)
rapid_parser.add_argument('--timeout', help='timeout (secs) after which to quit (default None)',
default=None, type=int)
rapid_parser.add_argument('--sleep', help='sleep time between loops', default=None, type=int)
single_parser.add_argument('-f', '--fw_id', help='specific fw_id to run in reservation mode',
default=None, type=int)
try:
import argcomplete
argcomplete.autocomplete(parser)
# This supports bash autocompletion. To enable this, pip install
# argcomplete, activate global completion, or add
# eval "$(register-python-argcomplete qlaunch)"
# into your .bash_profile or .bashrc
except ImportError:
pass
args = parser.parse_args()
if args.remote_host and not HAS_FABRIC:
print("Remote options require the Fabric package v2+ to be installed!")
sys.exit(-1)
if args.remote_setup and args.remote_host:
for h in args.remote_host:
conf = fabric.Configuration()
conf.run.shell = args.remote_shell
connect_kwargs = {'password': args.remote_password}
if args.remote_keyfile:
connect_kwargs["key_filename"] = args.remote_keyfile
with fabric.Connection(
host=h,
user=args.remote_user,
config=fabric.Config({'run': {'shell': args.remote_shell}}),
connect_kwargs=connect_kwargs) as conn:
for r in args.remote_config_dir:
r = os.path.expanduser(r)
conn.run("mkdir -p {}".format(r))
for f in os.listdir(args.config_dir):
if os.path.isfile(f):
conn.put(f, os.path.join(r, f))
non_default = []
for k in ["maxjobs_queue", "maxjobs_block", "nlaunches", "sleep"]:
v = getattr(args, k, None)
if v != rapid_parser.get_default(k):
non_default.append("--{} {}".format(k, v))
non_default = " ".join(non_default)
pre_non_default = []
for k in ["silencer", "reserve"]:
v = getattr(args, k, None)
if v:
pre_non_default.append("--%s" % k)
pre_non_default = " ".join(pre_non_default)
interval = args.daemon
while True:
if args.remote_host:
connect_kwargs = {'password': args.remote_password}
if args.remote_keyfile:
connect_kwargs["key_filename"] = args.remote_keyfile
for h in args.remote_host:
with fabric.Connection(
host=h,
user=args.remote_user,
config=fabric.Config({'run': {'shell': args.remote_shell}}),
connect_kwargs=connect_kwargs) as conn:
for r in args.remote_config_dir:
r = os.path.expanduser(r)
with conn.cd(r):
conn.run("qlaunch {} {} {}".format(
pre_non_default, args.command, non_default))
else:
do_launch(args)
if interval > 0:
print("Next run in {} seconds... Press Ctrl-C to exit at any "
"time.".format(interval))
time.sleep(args.daemon)
else:
break
import fabric
import invoke
import getpass
import shlex
import sys
new_password = getpass.getpass()
connect_kwargs = {'password': new_password}
config = fabric.Config(
overrides={'sudo': {'password': new_password}}
)
c = fabric.Connection(
'shl1.inf.susx.ac.uk', config=config, connect_kwargs=connect_kwargs
)
print("testing connection")
c.sudo("/bin/true")
print("success")
LXC_BINARY_PATH = "/var/lib/snapd/snap/bin/lxc"
STORAGE_POOL_NAME = "main"
BASE_IMAGE_NAME = "images:ubuntu/bionic/amd64"
CONTAINER_NAME = "demo"
def sudo_template(c, tmpl, *rest):
quoted = [shlex.quote(s) for s in rest]
expanded = tmpl.format(*quoted)
c.sudo(expanded)
