def run(self, experiment):
assert Agent().get_keys(
), "Add your private key to ssh agent using 'ssh-add' command"
# configure fabric
slurm_url = experiment.pop(
'slurm_url', '{}@{}'.format(PLGRID_USERNAME, PLGRID_HOST))
env['host_string'] = slurm_url
slurm_scratch_dir = Path(self._fabric_run('echo $SCRATCH'))
experiment = ExperimentRunOnSlurm(slurm_scratch_dir=slurm_scratch_dir,
slurm_url=slurm_url,
**filter_only_attr(
ExperimentRunOnSlurm,
experiment))
LOGGER.debug('Configuration: {}'.format(experiment))
self.ensure_directories(experiment)
script_path = self.deploy_code(experiment)
SCmd = {
'sbatch': SBatchWrapperCmd,
'srun': SRunWrapperCmd
}[experiment.cmd_type]
cmd = SCmd(experiment=experiment, script_path=script_path)
self._fabric_run(cmd.command)
def _prepare_ssh_pkey(self):
"""Tries to load config.ssh_key from file, for later use. Doesn't do that, it there are keys in the agent."""
from paramiko.agent import Agent
from paramiko import RSAKey, DSSKey
import getpass
if config.ssh_key is None:
# user didn't configure a specific ssh_key, so nothing to do for us
return None
agent = Agent()
if len(agent.get_keys()) > 0 and config.ssh_key_ignore_if_agent:
print "I: Running ssh-agent found, not loading ssh-key %s" % config.ssh_key
return None
# try to load the key from file
password = None
need_password = False
saved_exception = None
for pkey_class in (DSSKey, RSAKey):
try:
return pkey_class.from_private_key_file(config.ssh_key,
password=password)
except paramiko.PasswordRequiredException:
need_password = True
break
except paramiko.SSHException, e:
saved_exception = e
def __loadFromAgent(self):
agent = Agent()
keys = agent.get_keys()
if len(keys) == 0:
raise KeyError("No keys found in the ssh agent. Did you add them with ssh-add?")
return self.__pick_key(keys)
def _build_request(endpoint, get_params, post_params):
"""Wrap request data in an urllib Request instance
Aside from preparing the get and post data for transport, this function
authenticates the request using either an auth token or ssh keys.
Returns an urllib Request.
"""
if post_params:
post_data = json.dumps(post_params, default=json_encode_extra)
else:
post_data = None
timestamp = int(time.time())
headers = {
'Content-Encoding': 'application/x-json',
'X-Timestamp': str(timestamp),
'X-API-Version': '.'.join(str(v) for v in VERSION),
}
if Settings.auth_key:
headers['X-PublicKeys'] = Settings.auth_key.get_base64()
headers['X-Signatures'] = calc_signature(
Settings.auth_key, timestamp, post_data
)
elif Settings.auth_token:
headers['X-Application'] = calc_app_id(Settings.auth_token)
headers['X-SecurityToken'] = calc_security_token(
Settings.auth_token, timestamp, post_data
)
else:
try:
agent = Agent()
agent_keys = filter(signable, agent.get_keys())
except SSHException:
raise AuthenticationError('No token and ssh agent found')
if not agent_keys:
raise AuthenticationError('No token and ssh agent keys found')
key_signatures = {
key.get_base64(): calc_signature(key, timestamp, post_data)
for key in agent_keys
}
headers['X-PublicKeys'] = ','.join(key_signatures.keys())
headers['X-Signatures'] = ','.join(key_signatures.values())
if not Settings.base_url:
raise ConfigurationError(
'Environment variable SERVERADMIN_BASE_URL not set'
)
url = Settings.base_url + endpoint
if get_params:
url += '?' + urlencode(get_params)
if post_data:
post_data = post_data.encode('utf8')
return Request(url, post_data, headers)
def _smart_card(self, agent_request, pin=''):
""" Performs ssh-agent actions on smart cards using paramiko """
agent = Agent()
msg = Message()
msg.add_byte(agent_request)
msg.add_string(self._pkcs11_so)
msg.add_string(pin)
return (agent._send_message(msg))[0]
def __init__(self, transport: Transport) -> None:
self.agents: List[Union[Agent, AgentClientProxy]] = []
self.transport = transport
a = AgentServerProxy(self.transport)
os.environ.update(a.get_env())
a.connect()
self.agent = Agent()
self.keys = self.agent.get_keys()[:]
self.agents.append(self.agent)
def auth_agent():
agent = Agent()
agent_keys = agent.get_keys()
log("agent keys: %s", agent_keys)
if agent_keys:
for agent_key in agent_keys:
log("trying ssh-agent key '%s'", keymd5(agent_key))
try:
transport.auth_publickey(username, agent_key)
if transport.is_authenticated():
log("authenticated using agent and key '%s'", keymd5(agent_key))
break
except SSHException:
log("agent key '%s' rejected", keymd5(agent_key), exc_info=True)
if not transport.is_authenticated():
log.info("agent authentication failed, tried %i key%s", len(agent_keys), engs(agent_keys))
def _auth(self, username, password, pkey, key_filenames, allow_agent,
look_for_keys, gss_auth, gss_kex, gss_deleg_creds, gss_host):
"""
Try, in order:
- The key(s) passed in, if one was passed in.
- Any key we can find through an SSH agent (if allowed).
- Any "id_rsa", "id_dsa" or "id_ecdsa" key discoverable in ~/.ssh/
(if allowed).
- Plain username/password auth, if a password was given.
(The password might be needed to unlock a private key, or for
two-factor authentication [for which it is required].)
"""
saved_exception = None
two_factor = False
allowed_types = set()
two_factor_types = set(['keyboard-interactive', 'password'])
# If GSS-API support and GSS-PI Key Exchange was performed, we attempt
# authentication with gssapi-keyex.
if gss_kex and self._transport.gss_kex_used:
try:
self._transport.auth_gssapi_keyex(username)
return
except Exception as e:
saved_exception = e
# Try GSS-API authentication (gssapi-with-mic) only if GSS-API Key
# Exchange is not performed, because if we use GSS-API for the key
# exchange, there is already a fully established GSS-API context, so
# why should we do that again?
if gss_auth:
try:
return self._transport.auth_gssapi_with_mic(
username,
gss_host,
gss_deleg_creds,
)
except Exception as e:
saved_exception = e
if pkey is not None:
try:
self._log(
DEBUG,
'Trying SSH key %s' % hexlify(pkey.get_fingerprint()))
allowed_types = set(
self._transport.auth_publickey(username, pkey))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
except SSHException as e:
saved_exception = e
if not two_factor:
for key_filename in key_filenames:
for pkey_class in (RSAKey, DSSKey, ECDSAKey, Ed25519Key):
try:
key = self._key_from_filepath(
key_filename,
pkey_class,
password,
)
allowed_types = set(
self._transport.auth_publickey(username, key))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if not two_factor and allow_agent:
if self._agent is None:
self._agent = Agent()
for key in self._agent.get_keys():
try:
self._log(
DEBUG, 'Trying SSH agent key %s' %
hexlify(key.get_fingerprint()))
# for 2-factor auth a successfully auth'd key password
# will return an allowed 2fac auth method
allowed_types = set(
self._transport.auth_publickey(username, key))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if not two_factor:
keyfiles = []
for keytype, name in [
(RSAKey, "rsa"),
(DSSKey, "dsa"),
(ECDSAKey, "ecdsa"),
(Ed25519Key, "ed25519"),
]:
# ~/ssh/ is for windows
for directory in [".ssh", "ssh"]:
full_path = os.path.expanduser("~/%s/id_%s" %
(directory, name))
if os.path.isfile(full_path):
# TODO: only do this append if below did not run
keyfiles.append((keytype, full_path))
if os.path.isfile(full_path + '-cert.pub'):
keyfiles.append((keytype, full_path + '-cert.pub'))
if not look_for_keys:
keyfiles = []
for pkey_class, filename in keyfiles:
try:
key = self._key_from_filepath(
filename,
pkey_class,
password,
)
# for 2-factor auth a successfully auth'd key will result
# in ['password']
allowed_types = set(
self._transport.auth_publickey(username, key))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
break
except (SSHException, IOError) as e:
saved_exception = e
if password is not None:
try:
self._transport.auth_password(username, password)
return
except SSHException as e:
saved_exception = e
elif two_factor:
try:
self._transport.auth_interactive_dumb(username)
return
except SSHException as e:
saved_exception = e
# if we got an auth-failed exception earlier, re-raise it
if saved_exception is not None:
raise saved_exception
raise SSHException('No authentication methods available')
def _auth(self, username, password, pkey, key_filenames, allow_agent,
look_for_keys, gss_auth, gss_kex, gss_deleg_creds, gss_host):
"""
Try, in order:
- The key passed in, if one was passed in.
- Any key we can find through an SSH agent (if allowed).
- Any "id_rsa", "id_dsa" or "id_ecdsa" key discoverable in ~/.ssh/
(if allowed).
- Plain username/password auth, if a password was given.
(The password might be needed to unlock a private key, or for
two-factor authentication [for which it is required].)
"""
saved_exception = None
two_factor = False
allowed_types = []
# If GSS-API support and GSS-PI Key Exchange was performed, we attempt
# authentication with gssapi-keyex.
if gss_kex and self._transport.gss_kex_used:
try:
self._transport.auth_gssapi_keyex(username)
return
except Exception as e:
saved_exception = e
# Try GSS-API authentication (gssapi-with-mic) only if GSS-API Key
# Exchange is not performed, because if we use GSS-API for the key
# exchange, there is already a fully established GSS-API context, so
# why should we do that again?
if gss_auth:
try:
self._transport.auth_gssapi_with_mic(username, gss_host,
gss_deleg_creds)
return
except Exception as e:
saved_exception = e
if pkey is not None:
try:
self._log(
DEBUG,
'Trying SSH key %s' % hexlify(pkey.get_fingerprint()))
allowed_types = self._transport.auth_publickey(username, pkey)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
except SSHException as e:
saved_exception = e
if not two_factor:
for key_filename in key_filenames:
for pkey_class in (RSAKey, DSSKey, ECDSAKey):
try:
key = pkey_class.from_private_key_file(
key_filename, password)
self._log(
DEBUG, 'Trying key %s from %s' %
(hexlify(key.get_fingerprint()), key_filename))
self._transport.auth_publickey(username, key)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if not two_factor and allow_agent:
if self._agent is None:
self._agent = Agent()
for key in self._agent.get_keys():
try:
self._log(
DEBUG, 'Trying SSH agent key %s' %
hexlify(key.get_fingerprint()))
# for 2-factor auth a successfully auth'd key will result in ['password']
allowed_types = self._transport.auth_publickey(
username, key)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if not two_factor:
keyfiles = []
rsa_key = os.path.expanduser('~/.ssh/id_rsa')
dsa_key = os.path.expanduser('~/.ssh/id_dsa')
ecdsa_key = os.path.expanduser('~/.ssh/id_ecdsa')
if os.path.isfile(rsa_key):
keyfiles.append((RSAKey, rsa_key))
if os.path.isfile(dsa_key):
keyfiles.append((DSSKey, dsa_key))
if os.path.isfile(ecdsa_key):
keyfiles.append((ECDSAKey, ecdsa_key))
# look in ~/ssh/ for windows users:
rsa_key = os.path.expanduser('~/ssh/id_rsa')
dsa_key = os.path.expanduser('~/ssh/id_dsa')
ecdsa_key = os.path.expanduser('~/ssh/id_ecdsa')
if os.path.isfile(rsa_key):
keyfiles.append((RSAKey, rsa_key))
if os.path.isfile(dsa_key):
keyfiles.append((DSSKey, dsa_key))
if os.path.isfile(ecdsa_key):
keyfiles.append((ECDSAKey, ecdsa_key))
if not look_for_keys:
keyfiles = []
for pkey_class, filename in keyfiles:
try:
key = pkey_class.from_private_key_file(filename, password)
self._log(
DEBUG, 'Trying discovered key %s in %s' %
(hexlify(key.get_fingerprint()), filename))
# for 2-factor auth a successfully auth'd key will result in ['password']
allowed_types = self._transport.auth_publickey(
username, key)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
break
except (SSHException, IOError) as e:
saved_exception = e
if password is not None:
try:
self._transport.auth_password(username, password)
return
except SSHException as e:
saved_exception = e
elif two_factor:
raise SSHException('Two-factor authentication requires a password')
# if we got an auth-failed exception earlier, re-raise it
if saved_exception is not None:
raise saved_exception
raise SSHException('No authentication methods available')
def _auth(
self,
username,
password,
pkey,
key_filenames,
allow_agent,
look_for_keys,
gss_auth,
gss_kex,
gss_deleg_creds,
gss_host,
passphrase,
):
"""
Try, in order:
- The key(s) passed in, if one was passed in.
- Any key we can find through an SSH agent (if allowed).
- Any "id_rsa", "id_dsa" or "id_ecdsa" key discoverable in ~/.ssh/
(if allowed).
- Plain username/password auth, if a password was given.
(The password might be needed to unlock a private key [if 'passphrase'
isn't also given], or for two-factor authentication [for which it is
required].)
"""
saved_exception = None
two_factor = False
allowed_types = set()
two_factor_types = {'keyboard-interactive', 'password'}
if passphrase is None and password is not None:
passphrase = password
# If GSS-API support and GSS-PI Key Exchange was performed, we attempt
# authentication with gssapi-keyex.
if gss_kex and self._transport.gss_kex_used:
try:
self._transport.auth_gssapi_keyex(username)
return
except Exception as e:
saved_exception = e
# Try GSS-API authentication (gssapi-with-mic) only if GSS-API Key
# Exchange is not performed, because if we use GSS-API for the key
# exchange, there is already a fully established GSS-API context, so
# why should we do that again?
if gss_auth:
try:
return self._transport.auth_gssapi_with_mic(
username,
gss_host,
gss_deleg_creds,
)
except Exception as e:
saved_exception = e
# detect what authentication methods the server supports
# mirrors what openssh client does
try:
self._transport.auth_none(username)
except BadAuthenticationType as e:
allowed_types = set(e.allowed_types)
else:
return # successful login with no auth
# some servers do not return allowed auth methods
if not allowed_types:
allowed_types = {'password', 'publickey'}
if pkey is not None and 'publickey' in allowed_types:
try:
self._log(
DEBUG, 'Trying SSH key {}'.format(
hexlify(pkey.get_fingerprint())))
allowed_types = set(
self._transport.auth_publickey(username, pkey))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
except SSHException as e:
saved_exception = e
if not two_factor and 'publickey' in allowed_types:
for key_filename in key_filenames:
for pkey_class in (RSAKey, DSSKey, ECDSAKey, Ed25519Key):
try:
key = self._key_from_filepath(
key_filename,
pkey_class,
passphrase,
)
allowed_types = set(
self._transport.auth_publickey(username, key))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if allow_agent and not two_factor and 'publickey' in allowed_types:
if self._agent is None:
self._agent = Agent()
for key in self._agent.get_keys():
try:
id_ = hexlify(key.get_fingerprint())
self._log(DEBUG, 'Trying SSH agent key {}'.format(id_))
allowed_types = set(
self._transport.auth_publickey(username, key))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if look_for_keys and not two_factor and 'publickey' in allowed_types:
keyfiles = []
for keytype, name in [
(RSAKey, "rsa"),
(DSSKey, "dsa"),
(ECDSAKey, "ecdsa"),
(Ed25519Key, "ed25519"),
]:
# ~/ssh/ is for windows
for directory in [".ssh", "ssh"]:
full_path = os.path.expanduser("~/{}/id_{}".format(
directory, name))
if os.path.isfile(full_path):
# TODO: only do this append if below did not run
keyfiles.append((keytype, full_path))
if os.path.isfile(full_path + '-cert.pub'):
keyfiles.append((keytype, full_path + '-cert.pub'))
for pkey_class, filename in keyfiles:
try:
key = self._key_from_filepath(
filename,
pkey_class,
passphrase,
)
allowed_types = set(
self._transport.auth_publickey(username, key))
two_factor = (allowed_types & two_factor_types)
if not two_factor:
return
break
except (SSHException, IOError) as e:
saved_exception = e
# possible two_factor second factors
# (allowed_types could have been updated, only if two_factor)
if password is not None and 'password' in allowed_types:
try:
self._log(DEBUG, 'Trying password')
allowed_types = set(
self._transport.auth_password(username, password))
two_factor = allowed_types & two_factor_types
if not two_factor:
return
except SSHException as e:
saved_exception = e
if 'keyboard-interactive' in allowed_types:
try:
self._log(DEBUG, 'Trying interactive auth')
self._transport.auth_interactive_dumb(username)
return
except SSHException as e:
saved_exception = e
# if we got an auth-failed exception earlier, re-raise it
if saved_exception is not None:
raise saved_exception
raise SSHException('No authentication methods available')
for key_filename in key_filenames:
for pkey_class in (RSAKey, DSSKey):
try:
key = pkey_class.from_private_key_file(
key_filename, password)
self._log(
DEBUG, 'Trying key %s from %s' %
(hexlify(key.get_fingerprint()), key_filename))
self._transport.auth_publickey(username, key)
return
except SSHException, e:
saved_exception = e
if allow_agent:
for key in Agent().get_keys():
try:
self._log(
DEBUG, 'Trying SSH agent key %s' %
hexlify(key.get_fingerprint()))
self._transport.auth_publickey(username, key)
return
except SSHException, e:
saved_exception = e
keyfiles = []
rsa_key = os.path.expanduser('~/.ssh/id_rsa')
dsa_key = os.path.expanduser('~/.ssh/id_dsa')
if os.path.isfile(rsa_key):
keyfiles.append((RSAKey, rsa_key))
if os.path.isfile(dsa_key):
def _auth(self, username, password, pkey, key_filenames, allow_agent,
look_for_keys):
"""
Try, in order:
- The key passed in, if one was passed in.
- Any key we can find through an SSH agent (if allowed).
- Any "id_rsa" or "id_dsa" key discoverable in ~/.ssh/ (if allowed).
- Plain username/password auth, if a password was given.
(The password might be needed to unlock a private key, or for
two-factor authentication [for which it is required].)
"""
saved_exception = None
two_factor = False
allowed_types = []
if pkey is not None:
try:
self._log(
DEBUG,
'Trying SSH key %s' % hexlify(pkey.get_fingerprint()))
allowed_types = self._transport.auth_publickey(username, pkey)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
except SSHException as e:
saved_exception = e
if not two_factor:
for key_filename in key_filenames:
for pkey_class in (RSAKey, DSSKey):
try:
key = pkey_class.from_private_key_file(
key_filename, password)
self._log(
DEBUG, 'Trying key %s from %s' %
(hexlify(key.get_fingerprint()), key_filename))
self._transport.auth_publickey(username, key)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if not two_factor and allow_agent:
if self._agent is None:
self._agent = Agent()
for key in self._agent.get_keys():
try:
self._log(
DEBUG, 'Trying SSH agent key %s' %
hexlify(key.get_fingerprint()))
# for 2-factor auth a successfully auth'd key will result in ['password']
allowed_types = self._transport.auth_publickey(
username, key)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
break
except SSHException as e:
saved_exception = e
if not two_factor:
keyfiles = []
rsa_key = os.path.expanduser('~/.ssh/id_rsa')
dsa_key = os.path.expanduser('~/.ssh/id_dsa')
if os.path.isfile(rsa_key):
keyfiles.append((RSAKey, rsa_key))
if os.path.isfile(dsa_key):
keyfiles.append((DSSKey, dsa_key))
# look in ~/ssh/ for windows users:
rsa_key = os.path.expanduser('~/ssh/id_rsa')
dsa_key = os.path.expanduser('~/ssh/id_dsa')
if os.path.isfile(rsa_key):
keyfiles.append((RSAKey, rsa_key))
if os.path.isfile(dsa_key):
keyfiles.append((DSSKey, dsa_key))
if not look_for_keys:
keyfiles = []
for pkey_class, filename in keyfiles:
try:
key = pkey_class.from_private_key_file(filename, password)
self._log(
DEBUG, 'Trying discovered key %s in %s' %
(hexlify(key.get_fingerprint()), filename))
# for 2-factor auth a successfully auth'd key will result in ['password']
allowed_types = self._transport.auth_publickey(
username, key)
two_factor = (allowed_types == ['password'])
if not two_factor:
return
break
except (SSHException, IOError) as e:
saved_exception = e
if password is not None:
try:
self._transport.auth_password(username, password)
return
except SSHException as e:
saved_exception = e
elif two_factor:
raise SSHException('Two-factor authentication requires a password')
# if we got an auth-failed exception earlier, re-raise it
if saved_exception is not None:
raise saved_exception
raise SSHException('No authentication methods available')
def connect(self):
try:
with self._lock:
if not self._trans or not self._trans.is_authenticated():
logger.debug("Opening ssh connection ... ")
keys = None
logger.debug("Trying ssh-agent ... ")
drm4g_agent = drm4g.commands.Agent()
drm4g_agent.start()
drm4g_agent.update_agent_env()
# paramiko agent
agent = Agent()
keys = agent.get_keys()
if not keys:
logger.debug("Error trying to connect to '%s'" %
self.frontend)
logger.debug(
"Impossible to load '%s' key from the ssh-agent" %
self.private_key)
try:
status_ssh_agent = agent._conn
except Exception as err:
logger.warning(
"Probably you are using paramiko version <= 1.7.7.2 : %s "
% err)
status_ssh_agent = agent.conn
if not status_ssh_agent:
logger.warning("'ssh-agent' is not running")
else:
if agent.get_keys():
logger.warning(
"ssh-agent is running but none of the keys have been accepted"
"by remote frontend %s." % self.frontend)
else:
logger.debug(
"'ssh-agent' is running but without any keys"
)
if self.private_key:
logger.debug("Trying '%s' key ... " % self.private_key)
private_key_path = expanduser(self.private_key)
if (not exists(private_key_path)) and (
not 'PRIVATE KEY' in self.private_key):
output = "'%s'key does not exist" % private_key_path
raise ComException(output)
for pkey_class in (RSAKey, DSSKey):
try:
if 'PRIVATE KEY' in self.private_key:
import StringIO
key = pkey_class.from_private_key(
StringIO.StringIO(
self.private_key.strip("'")))
else:
key = pkey_class.from_private_key_file(
private_key_path)
keys = keys + (key, )
except Exception:
pass
if not keys:
output = "Impossible to load any keys"
logger.error(output)
raise ComException(output)
for key in keys:
try:
sock = socket.socket()
try:
sock.settimeout(SSH_CONNECT_TIMEOUT)
except:
output = "Timeout trying to connect to '%s'" % self.frontend
raise ComException(output)
logger.debug(
"Connecting to '%s' as user '%s' port '%s' ..."
% (self.frontend, self.username, self.port))
if ':' in self.frontend:
self.frontend, self.port = self.frontend.split(
':')
sock.connect((self.frontend, self.port))
self._trans = Transport(sock)
self._trans.connect(username=self.username,
pkey=key)
if self._trans.is_authenticated():
break
except socket.gaierror:
output = "Could not resolve hostname '%s' " % self.frontend
raise ComException(output)
except Exception as err:
logger.warning("Error connecting '%s': %s" %
(self.frontend, str(err)))
if not self._trans:
output = "Authentication failed for '%s'. Try to execute `ssh -vvv -p %d %s@%s` and see the response." % (
self.frontend, self.port, self.username, self.frontend)
raise ComException(output)
except ComException:
raise
except Exception as err:
if "No handlers could be found for logger" in str(err):
raise Exception(
"The connect function is the one causing problems : %s" %
str(err))
else:
raise
