def _apply_rubs(self, node, steps): """ Does the actual job over SSH :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param steps: the various steps of the rubbing :type steps: :class:`libcloud.compute.deployment.MultiStepDeployment` :return: ``True`` if everything went fine, ``False`` otherwise :rtype: ``bool`` """ if node is None or node.state != NodeState.RUNNING: logging.info("- skipped - node is not running") return False # select the address to use if len(node.public_ips) > 0: target_ip = node.public_ips[0] elif node.extra['ipv6']: target_ip = node.extra['ipv6'] else: target_ip = node.private_ips[0] # use libcloud to communicate with remote nodes session = SSHClient(hostname=target_ip, port=22, username='******', password=self.secret, key_files=None, timeout=9) try: session.connect() if self.engine.safeMode: logging.info("- no actual ssh interaction in safe mode") else: node = steps.run(node, session) except Exception as feedback: logging.info("Error: unable to rub '{}' at '{}'!".format( node.name, target_ip)) logging.error(str(feedback)) logging.info("- failed") result = False else: result = True try: session.close() except: pass return result
def test_usecase(self): client = SSHClient(hostname=self.ip_addresses[0], timeout=self.sshConnectTimeout) self.driver._ssh_client_connect(ssh_client=client, wait_period=self.wait_period, timeout=self.sshTimeout) cmd = '/bin/true' stdout, stderr, status = client.run(cmd) client.close() self.assertEqual(status, 0) print 'execute:', cmd
def _apply_rubs(self, node, steps): """ Does the actual job over SSH :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param steps: the various steps of the rubbing :type steps: :class:`libcloud.compute.deployment.MultiStepDeployment` :returns: ``bool`` - ``True`` if everything went fine, ``False`` otherwise """ # use libcloud to communicate with remote nodes session = SSHClient(hostname=node.private_ips[0], port=22, username='******', password=self.secret, key_files=None, timeout=15) try: session.connect() node = steps.run(node, session) except Exception as feedback: logging.info("Error: unable to rub '{}' at '{}'!".format(node.name, node.private_ips[0])) logging.info(str(feedback)) result = False else: result = True try: session.close() except: pass return result
def _apply_prepares(self, node, steps): """ Does the actual job over SSH :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param steps: the various steps of the preparing :type steps: :class:`libcloud.compute.deployment.MultiStepDeployment` :return: ``True`` if everything went fine, ``False`` otherwise :rtype: ``bool`` """ if node is None or node.state != NodeState.RUNNING: plogging.warning("- skipped - node is not running") return False # select the address to use if len(node.public_ips) > 0: target_ip = node.public_ips[0] elif node.extra['ipv6']: target_ip = node.extra['ipv6'] else: target_ip = node.private_ips[0] # guess location of user key path = os.path.expanduser('~/.ssh/id_rsa') # use libcloud to communicate with remote nodes session = SSHClient(hostname=target_ip, port=22, username=self.user, password=self.secret, key_files=path, timeout=9) try: session.connect() except Exception as feedback: plogging.error("Error: unable to prepare '{}' at '{}'!".format( node.name, target_ip)) plogging.error(str(feedback)) plogging.error("- failed") return False while True: try: if self.engine.safeMode: plogging.info("- skipped - no ssh interaction in safe mode") else: node = steps.run(node, session) except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue plogging.error("Error: unable to prepare '{}' at '{}'!".format( node.name, target_ip)) plogging.error(str(feedback)) plogging.error("- failed") result = False else: result = True break try: session.close() except: pass return result
def deploy_msd_to_node(libcloud_node, msd, private_key_path=None): ##msd = MultiStepDeployment(Scripts from water_machines above) logger.warn("TODO: REFACTOR AND TAKE OUT ec2-user literal") seed_profile = settings.operation_profile seed_profile = get_profile(seed_profile) pkey = seed_profile.keypair['local_path'] ssh_client = SSHClient(hostname=libcloud_node.private_ip[0], port=settings.SSH_PORT, username='******', password=None, key=pkey, timeout=int(settings.NETWORK_TIMEOUT),) attempts = 0 dns_attempts = 0 ##This begins a series of file placements for the masters subsequent deployment tasks in the init script. while True: time.sleep(5) if seed_profile.profile_name == "salt_master": dns_attempts += 1 logger.info("Number of attempts to connect: %s" % dns_attempts) try: logger.info("Attemping to connect to new node.") ssh_client.connect() logger.info("DNS SSH connection successful") except Exception as error: logger.info("DNS register ssh connection failed, trying again") dns_attempts += 1 if dns_attempts > 10: logger.error("DNS process failed to make a connection. Exiting.") break continue # salt-cloud files necessary for deployment for f in seed_profile.salt_cloud_vpc_files: try: cloud_files = FileDeployment(find_script(f), target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(f))) cloud_files.run(libcloud_node, ssh_client) logger.info("salt-cloud file %s placed in home directory" % f) except Exception as e: logger.error("could not place salt-cloud file: %s" % e) # places private key from path specified in keys.sh try: git_key = seed_profile.git_rsa_key git_key_file = FileDeployment(git_key, target= "/home/%s/%s" % (seed_profile.ami_user, os.path.basename(git_key))) git_key_file.run(libcloud_node, ssh_client) logger.info("Placed %s." % git_key_file.target) except Exception as e: logger.error("Could not place file: %s" % e) # places DNS registration files for the master to add itself to route 53 try: try_script = find_script(seed_profile.DNS_script) dns_file = FileDeployment(try_script, target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(try_script)) ) dns_file.run(libcloud_node, ssh_client) logger.info("Placed %s ." % dns_file.target) except Exception as e: logger.error("Could not place file: %s" % e) try: dns_command = find_script(seed_profile.DNS_command) domain = seed_profile.r53_domain r53_key = seed_profile.r53_key r53_secret = seed_profile.r53_secret w_command = open(dns_command, 'w') w_command.write("sudo python register_master_DNS.py '%(domain)s' '%(r53_key)s' '%(r53_secret)s'" % {'domain': domain, 'r53_key': r53_key, 'r53_secret': r53_secret}) w_command.close() c_deploy = FileDeployment(dns_command, target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(dns_command)) ) c_deploy.run(libcloud_node, ssh_client) r_command = open(dns_command, 'w') r_command.write(""" #This file get's blanked by the code to keep the keys out.\n echo 'The DNS register command did not make it to this file.'""" ) r_command.close() logger.info("The command file is in place") break except Exception as error: logger.error("Deployment of the DNS register file failed: %s", error) break else: print "%s isn't a master." % seed_profile.profile_name logger.warn("%s isn't a master." % seed_profile.profile_name) break ##This beings the deployment of init_scripts from water_machines while True: time.sleep(5) try: if ssh_client.connect() is True: # Deploy files to libcloud_node msd.run(libcloud_node, ssh_client) pubkey_file = find_script("master_public_keys.sh") ssh_key = get_public_key_from_file(pubkey_file) ssh_key.run(libcloud_node, ssh_client) for failed_step in msd.steps: try: execute_files_on_minion([failed_step], libcloud_node, ssh_client) except socket_timeout, timeout: logger.debug(timeout) # We'll have to have the minion ping the master # when it's alive and kicking so that we can confirm it's alive. # maybe via a webhook. # This happens when scripts you've implemented take to # long to complete. break except Exception, error: logger.error(error.message)
def deploy_node(self, **kwargs): """ Create a new node, and start deployment. Depends on a Provider Driver supporting either using a specific password or returning a generated password. This function may raise a L{DeploymentException}, if a create_node call was successful, but there is a later error (like SSH failing or timing out). This exception includes a Node object which you may want to destroy if incomplete deployments are not desirable. @keyword deploy: Deployment to run once machine is online and availble to SSH. @type deploy: L{Deployment} @keyword ssh_username: Optional name of the account which is used when connecting to SSH server (default is root) @type ssh_username: C{str} @keyword ssh_port: Optional SSH server port (default is 22) @type ssh_port: C{int} See L{NodeDriver.create_node} for more keyword args. >>> from libcloud.compute.drivers.dummy import DummyNodeDriver >>> from libcloud.deployment import ScriptDeployment, MultiStepDeployment >>> from libcloud.compute.base import NodeAuthSSHKey >>> driver = DummyNodeDriver(0) >>> key = NodeAuthSSHKey('...') # read from file >>> script = ScriptDeployment("yum -y install emacs strace tcpdump") >>> msd = MultiStepDeployment([key, script]) >>> def d(): ... try: ... node = driver.deploy_node(deploy=msd) ... except NotImplementedError: ... print "not implemented for dummy driver" >>> d() not implemented for dummy driver Deploy node is typically not overridden in subclasses. The existing implementation should be able to handle most such. """ # TODO: support ssh keys WAIT_PERIOD=3 password = None if 'generates_password' not in self.features["create_node"]: if 'password' not in self.features["create_node"]: raise NotImplementedError, \ 'deploy_node not implemented for this driver' if not kwargs.has_key('auth'): kwargs['auth'] = NodeAuthPassword(os.urandom(16).encode('hex')) password = kwargs['auth'].password node = self.create_node(**kwargs) try: if 'generates_password' in self.features["create_node"]: password = node.extra.get('password') start = time.time() end = start + (60 * 15) while time.time() < end: # need to wait until we get a public IP address. # TODO: there must be a better way of doing this time.sleep(WAIT_PERIOD) nodes = self.list_nodes() nodes = filter(lambda n: n.uuid == node.uuid, nodes) if len(nodes) == 0: raise DeploymentError(node, "Booted node[%s] is missing form list_nodes." % node) if len(nodes) > 1: raise DeploymentError(node, "Booted single node[%s], but multiple nodes have same UUID"% node) node = nodes[0] if node.public_ip is not None and node.public_ip != "" and node.state == NodeState.RUNNING: break ssh_username = kwargs.get('ssh_username', 'root') ssh_port = kwargs.get('ssh_port', 22) client = SSHClient(hostname=node.public_ip[0], port=ssh_port, username=ssh_username, password=password) laste = None while time.time() < end: laste = None try: client.connect() break except (IOError, socket.gaierror, socket.error), e: laste = e time.sleep(WAIT_PERIOD) if laste is not None: raise e tries = 3 while tries >= 0: try: n = kwargs["deploy"].run(node, client) client.close() break except Exception, e: tries -= 1 if tries == 0: raise client.connect()
def deploy_node(self, **kwargs): """ Create a new node, and start deployment. Depends on a Provider Driver supporting either using a specific password or returning a generated password. This function may raise a L{DeploymentException}, if a create_node call was successful, but there is a later error (like SSH failing or timing out). This exception includes a Node object which you may want to destroy if incomplete deployments are not desirable. @keyword deploy: Deployment to run once machine is online and availble to SSH. @type deploy: L{Deployment} @keyword ssh_username: Optional name of the account which is used when connecting to SSH server (default is root) @type ssh_username: C{str} @keyword ssh_port: Optional SSH server port (default is 22) @type ssh_port: C{int} @keyword ssh_timeout: Optional SSH connection timeout in seconds (default is None) @type ssh_timeout: C{float} @keyword auth: Initial authentication information for the node (optional) @type auth: L{NodeAuthSSHKey} or L{NodeAuthPassword} @keyword ssh_key: A path (or paths) to an SSH private key with which to attempt to authenticate. (optional) @type ssh_key: C{string} or C{list} of C{string}s @keyword max_tries: How many times to retry if a deployment fails before giving up (default is 3) @type max_tries: C{int} @keyword ssh_interface: The interface to wait for. Default is 'public_ips', other option is 'private_ips'. @type ssh_interface: C{str} See L{NodeDriver.create_node} for more keyword args. >>> from libcloud.compute.drivers.dummy import DummyNodeDriver >>> from libcloud.compute.deployment import ScriptDeployment >>> from libcloud.compute.deployment import MultiStepDeployment >>> from libcloud.compute.base import NodeAuthSSHKey >>> driver = DummyNodeDriver(0) >>> key = NodeAuthSSHKey('...') # read from file >>> script = ScriptDeployment("yum -y install emacs strace tcpdump") >>> msd = MultiStepDeployment([key, script]) >>> def d(): ... try: ... node = driver.deploy_node(deploy=msd) ... except NotImplementedError: ... print ("not implemented for dummy driver") >>> d() not implemented for dummy driver Deploy node is typically not overridden in subclasses. The existing implementation should be able to handle most such. """ if not libcloud.compute.ssh.have_paramiko: raise RuntimeError('paramiko is not installed. You can install ' + 'it using pip: pip install paramiko') password = None if 'create_node' not in self.features: raise NotImplementedError( 'deploy_node not implemented for this driver') elif 'generates_password' not in self.features["create_node"]: if 'password' not in self.features["create_node"] and \ 'ssh_key' not in self.features["create_node"]: raise NotImplementedError( 'deploy_node not implemented for this driver') if 'auth' not in kwargs: kwargs['auth'] = NodeAuthPassword(binascii.hexlify(os.urandom(16))) if 'ssh_key' not in kwargs: password = kwargs['auth'].password node = self.create_node(**kwargs) max_tries = kwargs.get('max_tries', 3) if 'generates_password' in self.features['create_node']: password = node.extra.get('password') try: # Wait until node is up and running and has IP assigned ssh_interface = kwargs.get('ssh_interface', 'public_ips') node, ip_addresses = self._wait_until_running(node=node, wait_period=3, timeout=NODE_ONLINE_WAIT_TIMEOUT, ssh_interface=ssh_interface) if password: node.extra['password'] = password ssh_username = kwargs.get('ssh_username', 'root') ssh_port = kwargs.get('ssh_port', 22) ssh_timeout = kwargs.get('ssh_timeout', 10) ssh_key_file = kwargs.get('ssh_key', None) ssh_client = SSHClient(hostname=ip_addresses[0], port=ssh_port, username=ssh_username, password=password, key=ssh_key_file, timeout=ssh_timeout) # Connect to the SSH server running on the node ssh_client = self._ssh_client_connect(ssh_client=ssh_client, timeout=SSH_CONNECT_TIMEOUT) # Execute the deployment task self._run_deployment_script(task=kwargs['deploy'], node=node, ssh_client=ssh_client, max_tries=max_tries) except Exception: e = sys.exc_info()[1] raise DeploymentError(node=node, original_exception=e, driver=self) return node
def _apply_prepares(self, node, steps): """ Does the actual job over SSH :param node: the node to be polished :type node: :class:`libcloud.compute.base.Node` :param steps: the various steps of the preparing :type steps: ``list`` of ``dict`` :return: ``True`` if everything went fine, ``False`` otherwise :rtype: ``bool`` """ if node is None or node.state != NodeState.RUNNING: plogging.warning("- skipped - node is not running") return False # select the address to use if len(node.public_ips) > 0: target_ip = node.public_ips[0] elif node.extra['ipv6']: target_ip = node.extra['ipv6'] else: target_ip = node.private_ips[0] # use libcloud to communicate with remote nodes session = SSHClient(hostname=target_ip, port=22, username=self.user, password=self.secret, key_files=self.key_files, timeout=10) repeats = 0 while True: try: session.connect() break except Exception as feedback: repeats += 1 if repeats > 5: plogging.error("Error: can not connect to '{}'!".format( target_ip)) plogging.error("- failed to connect") return False plogging.debug(str(feedback)) plogging.debug("- connection {} failed, retrying".format(repeats)) time.sleep(10) continue while True: try: if self.engine.safeMode: plogging.info("- skipped - no ssh interaction in safe mode") else: for step in steps: plogging.info('- {}'.format(step['description'])) step['genius'].run(node, session) except Exception as feedback: if 'RESOURCE_BUSY' in str(feedback): time.sleep(10) continue plogging.error("Error: unable to prepare '{}' at '{}'!".format( node.name, target_ip)) plogging.error(str(feedback)) plogging.error("- failed") result = False else: result = True break try: session.close() except: pass return result
def deploy_node(self, **kwargs): """ Create a new node, and start deployment. Depends on a Provider Driver supporting either using a specific password or returning a generated password. This function may raise a L{DeploymentException}, if a create_node call was successful, but there is a later error (like SSH failing or timing out). This exception includes a Node object which you may want to destroy if incomplete deployments are not desirable. @keyword deploy: Deployment to run once machine is online and availble to SSH. @type deploy: L{Deployment} @keyword ssh_username: Optional name of the account which is used when connecting to SSH server (default is root) @type ssh_username: C{str} @keyword ssh_port: Optional SSH server port (default is 22) @type ssh_port: C{int} See L{NodeDriver.create_node} for more keyword args. >>> from libcloud.compute.drivers.dummy import DummyNodeDriver >>> from libcloud.deployment import ScriptDeployment, MultiStepDeployment >>> from libcloud.compute.base import NodeAuthSSHKey >>> driver = DummyNodeDriver(0) >>> key = NodeAuthSSHKey('...') # read from file >>> script = ScriptDeployment("yum -y install emacs strace tcpdump") >>> msd = MultiStepDeployment([key, script]) >>> def d(): ... try: ... node = driver.deploy_node(deploy=msd) ... except NotImplementedError: ... print "not implemented for dummy driver" >>> d() not implemented for dummy driver Deploy node is typically not overridden in subclasses. The existing implementation should be able to handle most such. """ # TODO: support ssh keys # FIX: this method is too long and complicated WAIT_PERIOD = 3 password = None if 'generates_password' not in self.features["create_node"]: if 'password' not in self.features["create_node"]: raise NotImplementedError, \ 'deploy_node not implemented for this driver' if not kwargs.has_key('auth'): kwargs['auth'] = NodeAuthPassword(os.urandom(16).encode('hex')) password = kwargs['auth'].password node = self.create_node(**kwargs) try: if 'generates_password' in self.features["create_node"]: password = node.extra.get('password') start = time.time() end = start + (60 * 15) # FIX: this should be soft-coded while time.time() < end: # need to wait until we get a public IP address. # TODO: there must be a better way of doing this time.sleep(WAIT_PERIOD) nodes = self.list_nodes() nodes = filter(lambda n: n.uuid == node.uuid, nodes) if len(nodes) == 0: raise DeploymentError(node, ("Booted node[%s] " % node + "is missing from list_nodes.")) if len(nodes) > 1: raise DeploymentError( node, ("Booted single node[%s], " % node + "but multiple nodes have same UUID")) node = nodes[0] if (node.public_ip is not None and node.public_ip != "" and node.state == NodeState.RUNNING): break ssh_username = kwargs.get('ssh_username', 'root') ssh_port = kwargs.get('ssh_port', 22) client = SSHClient(hostname=node.public_ip[0], port=ssh_port, username=ssh_username, password=password) laste = None while time.time() < end: laste = None try: client.connect() break except (IOError, socket.gaierror, socket.error), e: laste = e time.sleep(WAIT_PERIOD) if laste is not None: raise e tries = 3 while tries >= 0: try: n = kwargs["deploy"].run(node, client) client.close() break except Exception, e: tries -= 1 if tries == 0: raise client.connect()