def _apply_rubs(self, node, steps):
"""
Does the actual job over SSH
:param node: the node to be polished
:type node: :class:`libcloud.compute.base.Node`
:param steps: the various steps of the rubbing
:type steps: :class:`libcloud.compute.deployment.MultiStepDeployment`
:return: ``True`` if everything went fine, ``False`` otherwise
:rtype: ``bool``
"""
if node is None or node.state != NodeState.RUNNING:
logging.info("- skipped - node is not running")
return False
# select the address to use
if len(node.public_ips) > 0:
target_ip = node.public_ips[0]
elif node.extra['ipv6']:
target_ip = node.extra['ipv6']
else:
target_ip = node.private_ips[0]
# use libcloud to communicate with remote nodes
session = SSHClient(hostname=target_ip,
port=22,
username='******',
password=self.secret,
key_files=None,
timeout=9)
try:
session.connect()
if self.engine.safeMode:
logging.info("- no actual ssh interaction in safe mode")
else:
node = steps.run(node, session)
except Exception as feedback:
logging.info("Error: unable to rub '{}' at '{}'!".format(
node.name, target_ip))
logging.error(str(feedback))
logging.info("- failed")
result = False
else:
result = True
try:
session.close()
except:
pass
return result
def test_usecase(self):
client = SSHClient(hostname=self.ip_addresses[0],
timeout=self.sshConnectTimeout)
self.driver._ssh_client_connect(ssh_client=client,
wait_period=self.wait_period,
timeout=self.sshTimeout)
cmd = '/bin/true'
stdout, stderr, status = client.run(cmd)
client.close()
self.assertEqual(status, 0)
print 'execute:', cmd
def _apply_rubs(self, node, steps):
"""
Does the actual job over SSH
:param node: the node to be polished
:type node: :class:`libcloud.compute.base.Node`
:param steps: the various steps of the rubbing
:type steps: :class:`libcloud.compute.deployment.MultiStepDeployment`
:returns: ``bool``
- ``True`` if everything went fine, ``False`` otherwise
"""
# use libcloud to communicate with remote nodes
session = SSHClient(hostname=node.private_ips[0],
port=22,
username='******',
password=self.secret,
key_files=None,
timeout=15)
try:
session.connect()
node = steps.run(node, session)
except Exception as feedback:
logging.info("Error: unable to rub '{}' at '{}'!".format(node.name,
node.private_ips[0]))
logging.info(str(feedback))
result = False
else:
result = True
try:
session.close()
except:
pass
return result
def _apply_prepares(self, node, steps):
"""
Does the actual job over SSH
:param node: the node to be polished
:type node: :class:`libcloud.compute.base.Node`
:param steps: the various steps of the preparing
:type steps: :class:`libcloud.compute.deployment.MultiStepDeployment`
:return: ``True`` if everything went fine, ``False`` otherwise
:rtype: ``bool``
"""
if node is None or node.state != NodeState.RUNNING:
plogging.warning("- skipped - node is not running")
return False
# select the address to use
if len(node.public_ips) > 0:
target_ip = node.public_ips[0]
elif node.extra['ipv6']:
target_ip = node.extra['ipv6']
else:
target_ip = node.private_ips[0]
# guess location of user key
path = os.path.expanduser('~/.ssh/id_rsa')
# use libcloud to communicate with remote nodes
session = SSHClient(hostname=target_ip,
port=22,
username=self.user,
password=self.secret,
key_files=path,
timeout=9)
try:
session.connect()
except Exception as feedback:
plogging.error("Error: unable to prepare '{}' at '{}'!".format(
node.name, target_ip))
plogging.error(str(feedback))
plogging.error("- failed")
return False
while True:
try:
if self.engine.safeMode:
plogging.info("- skipped - no ssh interaction in safe mode")
else:
node = steps.run(node, session)
except Exception as feedback:
if 'RESOURCE_BUSY' in str(feedback):
time.sleep(10)
continue
plogging.error("Error: unable to prepare '{}' at '{}'!".format(
node.name, target_ip))
plogging.error(str(feedback))
plogging.error("- failed")
result = False
else:
result = True
break
try:
session.close()
except:
pass
return result
def deploy_msd_to_node(libcloud_node, msd, private_key_path=None):
##msd = MultiStepDeployment(Scripts from water_machines above)
logger.warn("TODO: REFACTOR AND TAKE OUT ec2-user literal")
seed_profile = settings.operation_profile
seed_profile = get_profile(seed_profile)
pkey = seed_profile.keypair['local_path']
ssh_client = SSHClient(hostname=libcloud_node.private_ip[0],
port=settings.SSH_PORT,
username='******',
password=None,
key=pkey,
timeout=int(settings.NETWORK_TIMEOUT),)
attempts = 0
dns_attempts = 0
##This begins a series of file placements for the masters subsequent deployment tasks in the init script.
while True:
time.sleep(5)
if seed_profile.profile_name == "salt_master":
dns_attempts += 1
logger.info("Number of attempts to connect: %s" % dns_attempts)
try:
logger.info("Attemping to connect to new node.")
ssh_client.connect()
logger.info("DNS SSH connection successful")
except Exception as error:
logger.info("DNS register ssh connection failed, trying again")
dns_attempts += 1
if dns_attempts > 10:
logger.error("DNS process failed to make a connection. Exiting.")
break
continue
# salt-cloud files necessary for deployment
for f in seed_profile.salt_cloud_vpc_files:
try:
cloud_files = FileDeployment(find_script(f),
target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(f)))
cloud_files.run(libcloud_node, ssh_client)
logger.info("salt-cloud file %s placed in home directory" % f)
except Exception as e:
logger.error("could not place salt-cloud file: %s" % e)
# places private key from path specified in keys.sh
try:
git_key = seed_profile.git_rsa_key
git_key_file = FileDeployment(git_key,
target= "/home/%s/%s" % (seed_profile.ami_user, os.path.basename(git_key)))
git_key_file.run(libcloud_node, ssh_client)
logger.info("Placed %s." % git_key_file.target)
except Exception as e:
logger.error("Could not place file: %s" % e)
# places DNS registration files for the master to add itself to route 53
try:
try_script = find_script(seed_profile.DNS_script)
dns_file = FileDeployment(try_script,
target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(try_script)) )
dns_file.run(libcloud_node, ssh_client)
logger.info("Placed %s ." % dns_file.target)
except Exception as e:
logger.error("Could not place file: %s" % e)
try:
dns_command = find_script(seed_profile.DNS_command)
domain = seed_profile.r53_domain
r53_key = seed_profile.r53_key
r53_secret = seed_profile.r53_secret
w_command = open(dns_command, 'w')
w_command.write("sudo python register_master_DNS.py '%(domain)s' '%(r53_key)s' '%(r53_secret)s'" %
{'domain': domain,
'r53_key': r53_key,
'r53_secret': r53_secret})
w_command.close()
c_deploy = FileDeployment(dns_command,
target="/home/%s/%s" % (seed_profile.ami_user,
os.path.basename(dns_command)) )
c_deploy.run(libcloud_node, ssh_client)
r_command = open(dns_command, 'w')
r_command.write("""
#This file get's blanked by the code to keep the keys out.\n
echo 'The DNS register command did not make it to this file.'""" )
r_command.close()
logger.info("The command file is in place")
break
except Exception as error:
logger.error("Deployment of the DNS register file failed: %s", error)
break
else:
print "%s isn't a master." % seed_profile.profile_name
logger.warn("%s isn't a master." % seed_profile.profile_name)
break
##This beings the deployment of init_scripts from water_machines
while True:
time.sleep(5)
try:
if ssh_client.connect() is True:
# Deploy files to libcloud_node
msd.run(libcloud_node, ssh_client)
pubkey_file = find_script("master_public_keys.sh")
ssh_key = get_public_key_from_file(pubkey_file)
ssh_key.run(libcloud_node, ssh_client)
for failed_step in msd.steps:
try:
execute_files_on_minion([failed_step], libcloud_node, ssh_client)
except socket_timeout, timeout:
logger.debug(timeout)
# We'll have to have the minion ping the master
# when it's alive and kicking so that we can confirm it's alive.
# maybe via a webhook.
# This happens when scripts you've implemented take to
# long to complete.
break
except Exception, error:
logger.error(error.message)
def deploy_node(self, **kwargs):
"""
Create a new node, and start deployment.
Depends on a Provider Driver supporting either using a specific password
or returning a generated password.
This function may raise a L{DeploymentException}, if a create_node
call was successful, but there is a later error (like SSH failing or
timing out). This exception includes a Node object which you may want
to destroy if incomplete deployments are not desirable.
@keyword deploy: Deployment to run once machine is online and availble to SSH.
@type deploy: L{Deployment}
@keyword ssh_username: Optional name of the account which is used when connecting to
SSH server (default is root)
@type ssh_username: C{str}
@keyword ssh_port: Optional SSH server port (default is 22)
@type ssh_port: C{int}
See L{NodeDriver.create_node} for more keyword args.
>>> from libcloud.compute.drivers.dummy import DummyNodeDriver
>>> from libcloud.deployment import ScriptDeployment, MultiStepDeployment
>>> from libcloud.compute.base import NodeAuthSSHKey
>>> driver = DummyNodeDriver(0)
>>> key = NodeAuthSSHKey('...') # read from file
>>> script = ScriptDeployment("yum -y install emacs strace tcpdump")
>>> msd = MultiStepDeployment([key, script])
>>> def d():
... try:
... node = driver.deploy_node(deploy=msd)
... except NotImplementedError:
... print "not implemented for dummy driver"
>>> d()
not implemented for dummy driver
Deploy node is typically not overridden in subclasses. The
existing implementation should be able to handle most such.
"""
# TODO: support ssh keys
WAIT_PERIOD=3
password = None
if 'generates_password' not in self.features["create_node"]:
if 'password' not in self.features["create_node"]:
raise NotImplementedError, \
'deploy_node not implemented for this driver'
if not kwargs.has_key('auth'):
kwargs['auth'] = NodeAuthPassword(os.urandom(16).encode('hex'))
password = kwargs['auth'].password
node = self.create_node(**kwargs)
try:
if 'generates_password' in self.features["create_node"]:
password = node.extra.get('password')
start = time.time()
end = start + (60 * 15)
while time.time() < end:
# need to wait until we get a public IP address.
# TODO: there must be a better way of doing this
time.sleep(WAIT_PERIOD)
nodes = self.list_nodes()
nodes = filter(lambda n: n.uuid == node.uuid, nodes)
if len(nodes) == 0:
raise DeploymentError(node, "Booted node[%s] is missing form list_nodes." % node)
if len(nodes) > 1:
raise DeploymentError(node, "Booted single node[%s], but multiple nodes have same UUID"% node)
node = nodes[0]
if node.public_ip is not None and node.public_ip != "" and node.state == NodeState.RUNNING:
break
ssh_username = kwargs.get('ssh_username', 'root')
ssh_port = kwargs.get('ssh_port', 22)
client = SSHClient(hostname=node.public_ip[0],
port=ssh_port, username=ssh_username,
password=password)
laste = None
while time.time() < end:
laste = None
try:
client.connect()
break
except (IOError, socket.gaierror, socket.error), e:
laste = e
time.sleep(WAIT_PERIOD)
if laste is not None:
raise e
tries = 3
while tries >= 0:
try:
n = kwargs["deploy"].run(node, client)
client.close()
break
except Exception, e:
tries -= 1
if tries == 0:
raise
client.connect()
def deploy_node(self, **kwargs):
"""
Create a new node, and start deployment.
Depends on a Provider Driver supporting either using a specific
password or returning a generated password.
This function may raise a L{DeploymentException}, if a create_node
call was successful, but there is a later error (like SSH failing or
timing out). This exception includes a Node object which you may want
to destroy if incomplete deployments are not desirable.
@keyword deploy: Deployment to run once machine is online and
availble to SSH.
@type deploy: L{Deployment}
@keyword ssh_username: Optional name of the account which is used
when connecting to
SSH server (default is root)
@type ssh_username: C{str}
@keyword ssh_port: Optional SSH server port (default is 22)
@type ssh_port: C{int}
@keyword ssh_timeout: Optional SSH connection timeout in seconds
(default is None)
@type ssh_timeout: C{float}
@keyword auth: Initial authentication information for the node
(optional)
@type auth: L{NodeAuthSSHKey} or L{NodeAuthPassword}
@keyword ssh_key: A path (or paths) to an SSH private key with which
to attempt to authenticate. (optional)
@type ssh_key: C{string} or C{list} of C{string}s
@keyword max_tries: How many times to retry if a deployment fails
before giving up (default is 3)
@type max_tries: C{int}
@keyword ssh_interface: The interface to wait for. Default is
'public_ips', other option is 'private_ips'.
@type ssh_interface: C{str}
See L{NodeDriver.create_node} for more keyword args.
>>> from libcloud.compute.drivers.dummy import DummyNodeDriver
>>> from libcloud.compute.deployment import ScriptDeployment
>>> from libcloud.compute.deployment import MultiStepDeployment
>>> from libcloud.compute.base import NodeAuthSSHKey
>>> driver = DummyNodeDriver(0)
>>> key = NodeAuthSSHKey('...') # read from file
>>> script = ScriptDeployment("yum -y install emacs strace tcpdump")
>>> msd = MultiStepDeployment([key, script])
>>> def d():
... try:
... node = driver.deploy_node(deploy=msd)
... except NotImplementedError:
... print ("not implemented for dummy driver")
>>> d()
not implemented for dummy driver
Deploy node is typically not overridden in subclasses. The
existing implementation should be able to handle most such.
"""
if not libcloud.compute.ssh.have_paramiko:
raise RuntimeError('paramiko is not installed. You can install ' +
'it using pip: pip install paramiko')
password = None
if 'create_node' not in self.features:
raise NotImplementedError(
'deploy_node not implemented for this driver')
elif 'generates_password' not in self.features["create_node"]:
if 'password' not in self.features["create_node"] and \
'ssh_key' not in self.features["create_node"]:
raise NotImplementedError(
'deploy_node not implemented for this driver')
if 'auth' not in kwargs:
kwargs['auth'] = NodeAuthPassword(binascii.hexlify(os.urandom(16)))
if 'ssh_key' not in kwargs:
password = kwargs['auth'].password
node = self.create_node(**kwargs)
max_tries = kwargs.get('max_tries', 3)
if 'generates_password' in self.features['create_node']:
password = node.extra.get('password')
try:
# Wait until node is up and running and has IP assigned
ssh_interface = kwargs.get('ssh_interface', 'public_ips')
node, ip_addresses = self._wait_until_running(node=node,
wait_period=3, timeout=NODE_ONLINE_WAIT_TIMEOUT,
ssh_interface=ssh_interface)
if password:
node.extra['password'] = password
ssh_username = kwargs.get('ssh_username', 'root')
ssh_port = kwargs.get('ssh_port', 22)
ssh_timeout = kwargs.get('ssh_timeout', 10)
ssh_key_file = kwargs.get('ssh_key', None)
ssh_client = SSHClient(hostname=ip_addresses[0],
port=ssh_port, username=ssh_username,
password=password,
key=ssh_key_file,
timeout=ssh_timeout)
# Connect to the SSH server running on the node
ssh_client = self._ssh_client_connect(ssh_client=ssh_client,
timeout=SSH_CONNECT_TIMEOUT)
# Execute the deployment task
self._run_deployment_script(task=kwargs['deploy'],
node=node,
ssh_client=ssh_client,
max_tries=max_tries)
except Exception:
e = sys.exc_info()[1]
raise DeploymentError(node=node, original_exception=e, driver=self)
return node
def _apply_prepares(self, node, steps):
"""
Does the actual job over SSH
:param node: the node to be polished
:type node: :class:`libcloud.compute.base.Node`
:param steps: the various steps of the preparing
:type steps: ``list`` of ``dict``
:return: ``True`` if everything went fine, ``False`` otherwise
:rtype: ``bool``
"""
if node is None or node.state != NodeState.RUNNING:
plogging.warning("- skipped - node is not running")
return False
# select the address to use
if len(node.public_ips) > 0:
target_ip = node.public_ips[0]
elif node.extra['ipv6']:
target_ip = node.extra['ipv6']
else:
target_ip = node.private_ips[0]
# use libcloud to communicate with remote nodes
session = SSHClient(hostname=target_ip,
port=22,
username=self.user,
password=self.secret,
key_files=self.key_files,
timeout=10)
repeats = 0
while True:
try:
session.connect()
break
except Exception as feedback:
repeats += 1
if repeats > 5:
plogging.error("Error: can not connect to '{}'!".format(
target_ip))
plogging.error("- failed to connect")
return False
plogging.debug(str(feedback))
plogging.debug("- connection {} failed, retrying".format(repeats))
time.sleep(10)
continue
while True:
try:
if self.engine.safeMode:
plogging.info("- skipped - no ssh interaction in safe mode")
else:
for step in steps:
plogging.info('- {}'.format(step['description']))
step['genius'].run(node, session)
except Exception as feedback:
if 'RESOURCE_BUSY' in str(feedback):
time.sleep(10)
continue
plogging.error("Error: unable to prepare '{}' at '{}'!".format(
node.name, target_ip))
plogging.error(str(feedback))
plogging.error("- failed")
result = False
else:
result = True
break
try:
session.close()
except:
pass
return result
def deploy_node(self, **kwargs):
"""
Create a new node, and start deployment.
Depends on a Provider Driver supporting either using a specific password
or returning a generated password.
This function may raise a L{DeploymentException}, if a create_node
call was successful, but there is a later error (like SSH failing or
timing out). This exception includes a Node object which you may want
to destroy if incomplete deployments are not desirable.
@keyword deploy: Deployment to run once machine is online and availble to SSH.
@type deploy: L{Deployment}
@keyword ssh_username: Optional name of the account which is used when connecting to
SSH server (default is root)
@type ssh_username: C{str}
@keyword ssh_port: Optional SSH server port (default is 22)
@type ssh_port: C{int}
See L{NodeDriver.create_node} for more keyword args.
>>> from libcloud.compute.drivers.dummy import DummyNodeDriver
>>> from libcloud.deployment import ScriptDeployment, MultiStepDeployment
>>> from libcloud.compute.base import NodeAuthSSHKey
>>> driver = DummyNodeDriver(0)
>>> key = NodeAuthSSHKey('...') # read from file
>>> script = ScriptDeployment("yum -y install emacs strace tcpdump")
>>> msd = MultiStepDeployment([key, script])
>>> def d():
... try:
... node = driver.deploy_node(deploy=msd)
... except NotImplementedError:
... print "not implemented for dummy driver"
>>> d()
not implemented for dummy driver
Deploy node is typically not overridden in subclasses. The
existing implementation should be able to handle most such.
"""
# TODO: support ssh keys
# FIX: this method is too long and complicated
WAIT_PERIOD = 3
password = None
if 'generates_password' not in self.features["create_node"]:
if 'password' not in self.features["create_node"]:
raise NotImplementedError, \
'deploy_node not implemented for this driver'
if not kwargs.has_key('auth'):
kwargs['auth'] = NodeAuthPassword(os.urandom(16).encode('hex'))
password = kwargs['auth'].password
node = self.create_node(**kwargs)
try:
if 'generates_password' in self.features["create_node"]:
password = node.extra.get('password')
start = time.time()
end = start + (60 * 15) # FIX: this should be soft-coded
while time.time() < end:
# need to wait until we get a public IP address.
# TODO: there must be a better way of doing this
time.sleep(WAIT_PERIOD)
nodes = self.list_nodes()
nodes = filter(lambda n: n.uuid == node.uuid, nodes)
if len(nodes) == 0:
raise DeploymentError(node,
("Booted node[%s] " % node +
"is missing from list_nodes."))
if len(nodes) > 1:
raise DeploymentError(
node, ("Booted single node[%s], " % node +
"but multiple nodes have same UUID"))
node = nodes[0]
if (node.public_ip is not None and node.public_ip != ""
and node.state == NodeState.RUNNING):
break
ssh_username = kwargs.get('ssh_username', 'root')
ssh_port = kwargs.get('ssh_port', 22)
client = SSHClient(hostname=node.public_ip[0],
port=ssh_port,
username=ssh_username,
password=password)
laste = None
while time.time() < end:
laste = None
try:
client.connect()
break
except (IOError, socket.gaierror, socket.error), e:
laste = e
time.sleep(WAIT_PERIOD)
if laste is not None:
raise e
tries = 3
while tries >= 0:
try:
n = kwargs["deploy"].run(node, client)
client.close()
break
except Exception, e:
tries -= 1
if tries == 0:
raise
client.connect()
