Example #1
0
 def gevent_runner_func(self, hostname, ipmi, ip, username, password, supported_sensors, ipmi_state,
                        sel_event_log_list, option="key"):
     return_dict = dict()
     #self.log("info", "Gevent Thread created for %s" % hostname)
     try:
         sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
         sshclient.connect(ip, hostname, option)
         self.fetch_and_process_resource_info(hostname, ip, sshclient)
         self.fetch_and_process_network_info(hostname, ip, sshclient)
         self.fetch_and_process_disk_info(hostname, ip, sshclient)
         self.fetch_and_process_file_system_view(hostname, ip, sshclient)
         sshclient.close()
     except Exception as e:
         self.log("error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e))
         sshclient.close()
         pass
     try:
         return_dict["ipmi_status"] = \
             self.fetch_and_process_monitoring(hostname, ipmi, ip, username, password, supported_sensors)
         self.fetch_and_process_chassis(hostname, ipmi, ip, username, password)
         if sel_event_log_list:
             return_dict["sel_log"] = \
                 self.fetch_and_process_sel_logs(hostname, ipmi, username, password, sel_event_log_list)
         else:
             return_dict["sel_log"] = self.fetch_and_process_sel_logs(hostname, ipmi, username, password, [])
         if not ipmi_state and return_dict["ipmi_status"]:
             # Trigger REST API CALL to inventory for Server Hostname
             payload = dict()
             payload["id"] = str(hostname)
             self.send_run_inventory_request(self.smgr_ip, self.smgr_port, payload=payload)
         # self.log("info", "Payload return dict:\n" + str(return_dict))
         return return_dict
     except Exception as e:
         self.log("error", "Error in getting monitoring info through IPMI for server id: " + str(hostname) + " Error : " + str(e))
         pass
 def testDiskInfo(self):
     global package_installed
     sshclient = ServerMgrSSHClient(None)
     sshclient.filename = 'disk_info/disk_info_expected_output.txt'
     self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
     #Test the case where the diff between the previous and current iteration is sent
     sshclient.filename = 'disk_info/disk_info_diff_expected_output.txt'
     self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
     #Test the situation where the sysstat package is not installed on the target node
     package_installed = False
     self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
     sshclient.filename = 'disk_info/disk_info_wrong_output.txt'
     #Make sure the exception is raised. If the exception is not raised the test case will fail
     package_installed = True
     with self.assertRaises(Exception):
         self.ipmonitor.fetch_and_process_disk_info("dummy","dummy",sshclient)
Example #3
0
 def testDiskInfo(self):
     global package_installed
     sshclient = ServerMgrSSHClient(None)
     sshclient.filename = 'disk_info/disk_info_expected_output.txt'
     self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
     #Test the case where the diff between the previous and current iteration is sent
     sshclient.filename = 'disk_info/disk_info_diff_expected_output.txt'
     self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
     #Test the situation where the sysstat package is not installed on the target node
     package_installed = False
     self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
     sshclient.filename = 'disk_info/disk_info_wrong_output.txt'
     #Make sure the exception is raised. If the exception is not raised the test case will fail
     package_installed = True
     with self.assertRaises(Exception):
         self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy",
                                                    sshclient)
Example #4
0
 def gevent_runner_function(self,
                            action,
                            hostname,
                            ip=None,
                            ipmi=None,
                            username=None,
                            password=None,
                            option="key"):
     sshclient = None
     if action == "add" and ip and ipmi and username and password:
         try:
             sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
             sshclient.connect(ip, hostname, option)
             self.get_facter_info(hostname, ip, sshclient)
             self.get_cpu_info(hostname, ip, sshclient)
             self.get_ethernet_info(hostname, ip, sshclient)
             self.get_memory_info(hostname, ip, sshclient)
             sshclient.close()
         except Exception as e:
             if sshclient:
                 sshclient.close()
             self.log(
                 "error", "Gevent SSH Connect Exception for server id: " +
                 str(hostname) + " Error : " + str(e))
             pass
         try:
             self.get_fru_info(hostname, ipmi, username, password)
         except Exception as e:
             self.log(
                 "error",
                 "Error in getting Inventory Info through IPMI for server id: "
                 + str(hostname) + " Error : " + str(e))
             pass
     elif action == "delete":
         self.log(self.INFO, "Deleted info of server: %s" % hostname)
         self.delete_inventory_info(hostname)
 def gevent_runner_function(self, action, hostname, ip=None, ipmi=None, username=None, password=None, option="key"):
     sshclient = None
     try:
         if action == "add" and ip and ipmi and username and password:
             sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
             sshclient.connect(ip, hostname, option)
             self.get_fru_info(hostname, ipmi, username, password)
             self.get_facter_info(hostname, ip, sshclient)
             self.get_cpu_info(hostname, ip, sshclient)
             self.get_ethernet_info(hostname, ip, sshclient)
             self.get_memory_info(hostname, ip, sshclient)
             sshclient.close()
         elif action == "delete":
             self.log(self.INFO, "Deleted info of server: %s" % hostname)
             self.delete_inventory_info(hostname)
     except Exception as e:
         if sshclient:
             sshclient.close()
         self.log("error",
                  "Gevent SSH Connect Execption for server id: " + str(hostname) + " Error : " + str(e))
         pass
    def gevent_puppet_agent_action(self,server, serverDb, sm_args, action,access_method="key"):
        success = False
        tries = 0
        gevent.sleep(30)
        self._smgr_log.log("debug", "Going to %s the puppet agent on the server %s"% (action, str(server['id'])))
        while not success and tries < int(sm_args.puppet_agent_retry_count):
            try:
                tries += 1
                sshclient = ServerMgrSSHClient(serverdb=serverDb)
                sshclient.connect(str(server['ip_address']), str(server['id']), access_method)
                op = sshclient.exec_command('python -c "import platform; print platform.linux_distribution()"')
                self._smgr_log.log("debug", "OP is %s" %op)
                os_type = 'centos' if 'centos' in op.lower() else 'ubuntu'
                if os_type == 'centos':
                    enable_puppet_svc_cmd = "chkconfig puppet on"
                    disable_puppet_svc_cmd = "chkconfig puppet off"
                else:
                    enable_puppet_svc_cmd = " sed -i 's/START=.*$/START=yes/' /etc/default/puppet && "\
                                            "/usr/bin/puppet resource service puppet ensure=running enable=true "
                    disable_puppet_svc_cmd = " sed -i 's/START=.*$/START=no/' /etc/default/puppet && " \
                                             "/usr/bin/puppet resource service puppet ensure=stopped enable=false "

                self._smgr_log.log("debug", "PUPPET START Command is %s" %enable_puppet_svc_cmd)
                if action == "start":
                    output = sshclient.exec_command(enable_puppet_svc_cmd)
                    self._smgr_log.log("debug", "OUTPUT1 is %s" %output)
                    output = sshclient.exec_command("puppet agent --enable")
                    self._smgr_log.log("debug", "OUTPUT2 is %s" %output)
                    output = sshclient.exec_command("service puppet restart")
                    self._smgr_log.log("debug", "OUTPUT3 is %s" %output)

                    self._smgr_log.log("debug", "Successfully started the puppet agent on the server " + str(server['id']))
                    self._provision_immediately_after_reimage = False
                else:
                    output = sshclient.exec_command(disable_puppet_svc_cmd)
                    output = sshclient.exec_command("puppet agent --disable")
                    output = sshclient.exec_command("service puppet stop")
                    self._smgr_log.log("debug", "Successfully stopped the puppet agent on the server " + str(server['id']))
                success = True
                sshclient.close()
            except Exception as e:
                if action == "start":
                    servers = self._serverDb.get_server({"id": server['id']}, detail=True)
                    server_state = servers[0]['status']
                    if server_state == "reimage_started" or server_state == "restart_issued" \
                       or server_state == "reimage_completed" or server_state == "provision_issued":
                        self._provision_immediately_after_reimage = True 
                if sshclient:
                    sshclient.close()
                self._smgr_log.log(self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e))
            self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" %(action, str(server["id"]), str(tries)))
            gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds))
        if tries >= int(sm_args.puppet_agent_retry_count) and success is False:
            if action == "start":
                self._smgr_log.log(self._smgr_log.ERROR, "Starting the puppet agent failed on  " + str(server["id"]))
            else:
                self._smgr_log.log(self._smgr_log.ERROR, "Stopping the puppet agent failed on  " + str(server["id"]))
    def copy_ssh_keys_to_server(self, ip_address, server_id):
      self._smgr_log.log(self._smgr_log.DEBUG, "COPY-KEY: Server: " + str(server_id))

      tries = 0
      gevent.sleep(60)
      success = False
      while True:
        try:
          tries = tries + 1
          #avoid overflow
          if tries > 10000:
            tries = 10000

          # We keep trying infinitely for this, so if server is deleted before
          # this, we need to know about it and pull new_public_key
          servers = self._serverDb.get_server({"id": server_id}, detail=True)
          if not servers :
            self._smgr_log.log(self._smgr_log.DEBUG, "COPY-KEY: Server: " + str(server_id) + " NOT FOUND")
            return False

          server = servers[0]
          ssh_key = server["ssh_public_key"]
          #try to connect first if target node is up
          source_file = "/tmp/" + str(server_id) + ".pub"
          ssh = ServerMgrSSHClient(self._serverDb)
          ssh.connect(ip_address, server_id, option="password")

          subprocess.call(['mkdir', '-p', '/tmp'])
          with open("/tmp/" + str(server_id) + ".pub", 'w+') as content_file:
            content_file.write(str(ssh_key))
            content_file.close()

          key_dest_file = "/root/" + str(server_id) + ".pub"
          dest_file = "/root/.ssh/authorized_keys"
          ssh.exec_command("mkdir -p /root/.ssh/")
          ssh.exec_command("touch " + str(key_dest_file))
          ssh.exec_command("touch " + str(dest_file))

          if os.path.exists(source_file):
            # Copy Public key on Target Server
            #TODO: check if authrized keys are already available
            bytes_sent = ssh.copy(source_file, key_dest_file)
            cmd = "grep -q -f " + str(key_dest_file) + " " + str(dest_file) + ";" \
                  + " RETVAL=$? ; " \
                  + " if [[ $RETVAL -eq 1 ]]; then \
                        echo '' >> " + str(dest_file) + "; \
                        cat " + str(key_dest_file) + " >> " + str(dest_file) + ";\
                        echo '' >> " + str(dest_file) + "; fi; \
                      rm -f " + str(key_dest_file) + "; "

            self._smgr_log.log(self._smgr_log.DEBUG, cmd)
            ssh.exec_command(cmd)
          ssh.close()
          if os.path.exists(source_file):
            os.remove(source_file)
          msg =  "COPY-KEYS: %s bytes copied on %s: " %(str(bytes_sent), str(server_id))
          self._smgr_log.log(self._smgr_log.DEBUG, msg)
          success = True
          return success

        except Exception as e:
          msg = "COPY-KEYS: Host : %s Try: %d: ERROR Copying Keys: %s" % (str(server_id), tries, str(e))
          self._smgr_log.log(self._smgr_log.ERROR, msg)
          if os.path.exists(source_file):
            os.remove(source_file)
          if ssh:
            ssh.close()
          if tries >= 20:
            sleep_time = 120
          else:
            sleep_time = 30

          gevent.sleep(sleep_time)
 def reimage_run_inventory(self, ip, port, payload):
     success = False
     source_file = ""
     sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
     self._smgr_log.log("debug", "Running Reimage Inventory on  " + str(payload))
     if not self.inventory_config_set and not self.monitoring_config_set:
         return "No Inventory or Monitoring Configured"
     tries = 0
     gevent.sleep(60)
     while not success and tries < 10:
         try:
             tries += 1
             server = self._serverDb.get_server({"id": str(payload["id"])}, detail=True)
             if server and len(server) == 1:
                 server = server[0]
                 subprocess.call(['ssh-keygen', '-f', '/root/.ssh/known_hosts', '-R', str(server["ip_address"])])
                 subprocess.call(['mkdir', '-p', '/tmp'])
                 sshclient.connect(str(server["ip_address"]), str(server["id"]), "password")
                 match_dict = dict()
                 match_dict["id"] = str(payload["id"])
                 self._smgr_log.log(self._smgr_log.DEBUG, "Running inventory on " + str(payload["id"]) +
                                    ", try " + str(tries))
                 ssh_public_ket_str = str(server["ssh_public_key"])
                 with open("/tmp/" + str(payload["id"]) + ".pub", 'w+') as content_file:
                     content_file.write(ssh_public_ket_str)
                     content_file.close()
                 source_file = "/tmp/" + str(payload["id"]) + ".pub"
                 dest_file = "/root/.ssh/authorized_keys"
                 if os.path.exists(source_file):
                     sshclient.exec_command('mkdir -p /root/.ssh/')
                     sshclient.exec_command('touch /root/.ssh/authorized_keys')
                     bytes_sent = sshclient.copy(source_file, dest_file)
                     sshclient.exec_command("echo '' >> " + str(dest_file))
                 sshclient.close()
                 self._smgr_log.log(self._smgr_log.DEBUG, "SSH Keys copied on  " + str(payload["id"]) +
                                    ", try " + str(tries))
                 if os.path.exists(source_file):
                     os.remove(source_file)
                 success = True
             else:
                 self._smgr_log.log(self._smgr_log.ERROR, "Server Matching Server Id:  " + str(payload["id"]) +
                                    " not found. SSH Keys will not be copied. ")
                 tries = 10
                 if os.path.exists(source_file):
                     os.remove(source_file)
                 if sshclient:
                     sshclient.close()
                 success = False
         except Exception as e:
             if os.path.exists(source_file):
                 os.remove(source_file)
             if sshclient:
                 sshclient.close()
             gevent.sleep(30)
     if tries >= 10 and success is False:
         self._smgr_log.log(self._smgr_log.ERROR, "SSH Key copy failed on  " + str(payload["id"]))
     if success and self.inventory_config_set:
         try:
             url = "http://%s:%s/run_inventory" % (ip, port)
             payload = json.dumps(payload)
             headers = {'content-type': 'application/json'}
             resp = requests.post(url, headers=headers, timeout=5, data=payload)
             return resp.text
         except Exception as e:
             self._smgr_log.log("error", "Error running inventory on  " + str(payload) + " : " + str(e))
             return None
    def create_store_copy_ssh_keys(self, server_id, server_ip):

        # Create the Keys using Pycrypto
        self._smgr_log.log(self._smgr_log.DEBUG, "Copying keys for server: " + str(server_id))
        ssh_key = paramiko.RSAKey.generate(bits=2048)
        ssh_private_key_obj = StringIO.StringIO()
        ssh_key.write_private_key(ssh_private_key_obj)
        source_file = ""
        bytes_sent = 0
        try:
            subprocess.call(['mkdir', '-p', '/tmp'])
            # Save Public key on Target Server
            source_file = "/tmp/" + str(server_id) + ".pub"
            with open("/tmp/" + str(server_id) + ".pub", 'w+') as content_file:
                content_file.write("ssh-rsa " + str(ssh_key.get_base64()))
                content_file.close()
            ssh = ServerMgrSSHClient(self._serverDb)
            ssh.connect(server_ip, server_id, option="password")
            key_dest_file = "/root/" + str(server_id) + ".pub"
            dest_file = "/root/.ssh/authorized_keys"
            ssh.exec_command("mkdir -p /root/.ssh/")
            ssh.exec_command("touch " + str(key_dest_file))
            if os.path.exists(source_file):
                bytes_sent = ssh.copy(source_file, key_dest_file)
                ssh.exec_command("echo '' >> " + str(dest_file))
                ssh.exec_command("cat " + str(key_dest_file) + " >> " + str(dest_file))
                ssh.exec_command("echo '' >> " + str(dest_file))
                ssh.exec_command("rm -rf " + str(key_dest_file))
            # Update Server table with ssh public and private keys
            update = {'id': server_id,
                      'ssh_public_key': "ssh-rsa " + str(ssh_key.get_base64()),
                      'ssh_private_key': ssh_private_key_obj.getvalue()}
            self._serverDb.modify_server(update)
            ssh.close()
            if os.path.exists(source_file):
                os.remove(source_file)
            self._smgr_log.log(self._smgr_log.DEBUG, "Bytes copied in ssh_key copy: " + str(bytes_sent))
            return ssh_key
        except Exception as e:
            self._smgr_log.log(self._smgr_log.ERROR, "Error Creating/Copying Keys: " + str(server_id) + " : " + str(e))
            if os.path.exists(source_file):
                os.remove(source_file)
            # Update Server table with ssh public and private keys
            update = {'id': server_id,
                      'ssh_public_key': "ssh-rsa " + str(ssh_key.get_base64()),
                      'ssh_private_key': ssh_private_key_obj.getvalue()}
            self._serverDb.modify_server(update)
            return None
 def gevent_puppet_agent_action(self,server, serverDb, sm_args, action,access_method="key"):
     success = False
     tries = 0
     gevent.sleep(30)
     self._smgr_log.log("debug", "Going to %s the puppet agent on the server %s"% (action, str(server['id'])))
     while not success and tries < int(sm_args.puppet_agent_retry_count):
         try:
             tries += 1
             sshclient = ServerMgrSSHClient(serverdb=serverDb)
             sshclient.connect(str(server['ip_address']), str(server['id']), access_method)
             if action == "start":
                 output = sshclient.exec_command("sed -i 's/START=.*$/START=yes/' /etc/default/puppet")
                 output = sshclient.exec_command("service puppet start")
                 self._smgr_log.log("debug", "Successfully started the puppet agent on the server " + str(server['id']))
                 self._provision_immediately_after_reimage = False
             else:
                 output = sshclient.exec_command("sed -i 's/START=.*$/START=no/' /etc/default/puppet")
                 output = sshclient.exec_command("service puppet stop")
                 self._smgr_log.log("debug", "Successfully stopped the puppet agent on the server " + str(server['id']))
             success = True
             sshclient.close()
         except Exception as e:
             if action == "start":
                 server_state = server['status']
                 if server_state == "reimage_started" or server_state == "restart_issued" or server_state == "reimage_completed":
                     self._provision_immediately_after_reimage = True 
             if sshclient:
                 sshclient.close()
             self._smgr_log.log(self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e))
         self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" %(action, str(server["id"]), str(tries)))
         gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds))
     if tries >= int(sm_args.puppet_agent_retry_count) and success is False:
         if action == "start":
             self._smgr_log.log(self._smgr_log.ERROR, "Starting the puppet agent failed on  " + str(server["id"]))
         else:
             self._smgr_log.log(self._smgr_log.ERROR, "Stopping the puppet agent failed on  " + str(server["id"]))
Example #11
0
    def gevent_puppet_agent_action(self,
                                   server,
                                   serverDb,
                                   sm_args,
                                   action,
                                   access_method="key"):
        success = False
        tries = 0
        gevent.sleep(30)
        self._smgr_log.log(
            "debug", "Going to %s the puppet agent on the server %s" %
            (action, str(server['id'])))
        while not success and tries < int(sm_args.puppet_agent_retry_count):
            try:
                tries += 1
                sshclient = ServerMgrSSHClient(serverdb=serverDb)
                sshclient.connect(str(server['ip_address']), str(server['id']),
                                  access_method)
                op = sshclient.exec_command(
                    'python -c "import platform; print platform.linux_distribution()"'
                )
                self._smgr_log.log("debug", "OP is %s" % op)
                os_type = 'centos' if 'centos' in op.lower() else 'ubuntu'
                if os_type == 'centos':
                    enable_puppet_svc_cmd = "chkconfig puppet on"
                    disable_puppet_svc_cmd = "chkconfig puppet off"
                else:
                    enable_puppet_svc_cmd = " sed -i 's/START=.*$/START=yes/' /etc/default/puppet && "\
                                            "/usr/bin/puppet resource service puppet ensure=running enable=true "
                    disable_puppet_svc_cmd = " sed -i 's/START=.*$/START=no/' /etc/default/puppet && " \
                                             "/usr/bin/puppet resource service puppet ensure=stopped enable=false "

                self._smgr_log.log(
                    "debug",
                    "PUPPET START Command is %s" % enable_puppet_svc_cmd)
                if action == "start":
                    output = sshclient.exec_command(enable_puppet_svc_cmd)
                    self._smgr_log.log("debug", "OUTPUT1 is %s" % output)
                    output = sshclient.exec_command("puppet agent --enable")
                    self._smgr_log.log("debug", "OUTPUT2 is %s" % output)
                    output = sshclient.exec_command("service puppet restart")
                    self._smgr_log.log("debug", "OUTPUT3 is %s" % output)

                    self._smgr_log.log(
                        "debug",
                        "Successfully started the puppet agent on the server "
                        + str(server['id']))
                    self._provision_immediately_after_reimage = False
                else:
                    output = sshclient.exec_command(disable_puppet_svc_cmd)
                    output = sshclient.exec_command("puppet agent --disable")
                    output = sshclient.exec_command("service puppet stop")
                    self._smgr_log.log(
                        "debug",
                        "Successfully stopped the puppet agent on the server "
                        + str(server['id']))
                success = True
                sshclient.close()
            except Exception as e:
                if action == "start":
                    servers = self._serverDb.get_server({"id": server['id']},
                                                        detail=True)
                    server_state = servers[0]['status']
                    if server_state == "reimage_started" or server_state == "restart_issued" \
                       or server_state == "reimage_completed" or server_state == "provision_issued":
                        self._provision_immediately_after_reimage = True
                if sshclient:
                    sshclient.close()
                self._smgr_log.log(
                    self._smgr_log.ERROR,
                    "Gevent SSH Connect Exception for server id: " +
                    server['id'] + " Error : " + str(e))
            self._smgr_log.log(
                self._smgr_log.DEBUG,
                "Still trying to %s the puppet agent in the server %s, try %s"
                % (action, str(server["id"]), str(tries)))
            gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds))
        if tries >= int(sm_args.puppet_agent_retry_count) and success is False:
            if action == "start":
                self._smgr_log.log(
                    self._smgr_log.ERROR,
                    "Starting the puppet agent failed on  " +
                    str(server["id"]))
            else:
                self._smgr_log.log(
                    self._smgr_log.ERROR,
                    "Stopping the puppet agent failed on  " +
                    str(server["id"]))
Example #12
0
    def copy_ssh_keys_to_server(self, ip_address, server_id):
        self._smgr_log.log(self._smgr_log.DEBUG,
                           "COPY-KEY: Server: " + str(server_id))

        tries = 0
        gevent.sleep(60)
        success = False
        while True:
            try:
                tries = tries + 1
                #avoid overflow
                if tries > 10000:
                    tries = 10000

                # We keep trying infinitely for this, so if server is deleted before
                # this, we need to know about it and pull new_public_key
                servers = self._serverDb.get_server({"id": server_id},
                                                    detail=True)
                if not servers:
                    self._smgr_log.log(
                        self._smgr_log.DEBUG,
                        "COPY-KEY: Server: " + str(server_id) + " NOT FOUND")
                    return False

                server = servers[0]
                ssh_key = server["ssh_public_key"]
                #try to connect first if target node is up
                source_file = "/tmp/" + str(server_id) + ".pub"
                ssh = ServerMgrSSHClient(self._serverDb)
                ssh.connect(ip_address, server_id, option="password")

                subprocess.call(['mkdir', '-p', '/tmp'])
                with open("/tmp/" + str(server_id) + ".pub",
                          'w+') as content_file:
                    content_file.write(str(ssh_key))
                    content_file.close()

                key_dest_file = "/root/" + str(server_id) + ".pub"
                dest_file = "/root/.ssh/authorized_keys"
                ssh.exec_command("mkdir -p /root/.ssh/")
                ssh.exec_command("touch " + str(key_dest_file))
                ssh.exec_command("touch " + str(dest_file))

                if os.path.exists(source_file):
                    # Copy Public key on Target Server
                    #TODO: check if authrized keys are already available
                    bytes_sent = ssh.copy(source_file, key_dest_file)
                    cmd = "grep -q -f " + str(key_dest_file) + " " + str(dest_file) + ";" \
                          + " RETVAL=$? ; " \
                          + " if [[ $RETVAL -eq 1 ]]; then \
                        echo '' >> "                                             + str(dest_file) + "; \
                        cat "                                      + str(key_dest_file) + " >> " + str(dest_file) + ";\
                        echo '' >> "                                             + str(dest_file) + "; fi; \
                      rm -f "                                      + str(key_dest_file) + "; "

                    self._smgr_log.log(self._smgr_log.DEBUG, cmd)
                    ssh.exec_command(cmd)
                ssh.close()
                if os.path.exists(source_file):
                    os.remove(source_file)
                msg = "COPY-KEYS: %s bytes copied on %s: " % (str(bytes_sent),
                                                              str(server_id))
                self._smgr_log.log(self._smgr_log.DEBUG, msg)
                success = True
                return success

            except Exception as e:
                msg = "COPY-KEYS: Host : %s Try: %d: ERROR Copying Keys: %s" % (
                    str(server_id), tries, str(e))
                self._smgr_log.log(self._smgr_log.ERROR, msg)
                if os.path.exists(source_file):
                    os.remove(source_file)
                if ssh:
                    ssh.close()
                if tries >= 20:
                    sleep_time = 120
                else:
                    sleep_time = 30

                gevent.sleep(sleep_time)
Example #13
0
 def gevent_puppet_agent_action(self,server, serverDb, sm_args, action,access_method="key"):
     success = False
     tries = 0
     gevent.sleep(30)
     pdb.set_trace()
     self._smgr_log.log("debug", "Going to %s the puppet agent on the server %s"% (action, str(server['id'])))
     while not success and tries < int(sm_args.puppet_agent_retry_count):
         try:
             tries += 1
             sshclient = ServerMgrSSHClient(serverdb=serverDb)
             sshclient.connect(str(server['ip_address']), str(server['id']), access_method)
             if action == "start":
                 output = sshclient.exec_command("sed -i 's/START=.*$/START=yes/' /etc/default/puppet")
                 output = sshclient.exec_command("service puppet start")
                 self._smgr_log.log("debug", "Successfully started the puppet agent on the server " + str(server['id']))
                 self._provision_immediately_after_reimage = False
             else:
                 output = sshclient.exec_command("sed -i 's/START=.*$/START=no/' /etc/default/puppet")
                 output = sshclient.exec_command("service puppet stop")
                 self._smgr_log.log("debug", "Successfully stopped the puppet agent on the server " + str(server['id']))
             success = True
             sshclient.close()
         except Exception as e:
             if action == "start":
                 server_state = server['status']
                 if server_state == "reimage_started" or server_state == "restart_issued" or server_state == "reimage_completed":
                     self._provision_immediately_after_reimage = True 
             if sshclient:
                 sshclient.close()
             self._smgr_log.log(self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e))
         self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" %(action, str(server["id"]), str(tries)))
         gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds))
     if tries >= int(sm_args.puppet_agent_retry_count) and success is False:
         if action == "start":
             self._smgr_log.log(self._smgr_log.ERROR, "Starting the puppet agent failed on  " + str(server["id"]))
         else:
             self._smgr_log.log(self._smgr_log.ERROR, "Stopping the puppet agent failed on  " + str(server["id"]))
Example #14
0
 def copy_ssh_keys_to_servers(self, ip, port, payload, sm_args=None):
     success = False
     source_file = ""
     sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
     self._smgr_log.log("debug", "Copying the ssh keys  " + str(payload))
     tries = 0
     gevent.sleep(60)
     while not success and tries < 10:
         try:
             tries += 1
             server = self._serverDb.get_server({"id": str(payload["id"])}, detail=True)
             if server and len(server) == 1:
                 server = server[0]
                 subprocess.call(['ssh-keygen', '-f', '/root/.ssh/known_hosts', '-R', str(server["ip_address"])])
                 subprocess.call(['mkdir', '-p', '/tmp'])
                 sshclient.connect(str(server["ip_address"]), str(server["id"]), "password")
                 match_dict = dict()
                 match_dict["id"] = str(payload["id"])
                 self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to copy the ssh keys " + str(payload["id"]) +
                                    ", try " + str(tries))
                 ssh_public_ket_str = str(server["ssh_public_key"])
                 with open("/tmp/" + str(payload["id"]) + ".pub", 'w+') as content_file:
                     content_file.write(ssh_public_ket_str)
                     content_file.close()
                 source_file = "/tmp/" + str(payload["id"]) + ".pub"
                 dest_file = "/root/.ssh/authorized_keys"
                 if os.path.exists(source_file):
                     sshclient.exec_command('mkdir -p /root/.ssh/')
                     sshclient.exec_command('touch /root/.ssh/authorized_keys')
                     bytes_sent = sshclient.copy(source_file, dest_file)
                     sshclient.exec_command("echo '' >> " + str(dest_file))
                 sshclient.close()
                 self._smgr_log.log(self._smgr_log.DEBUG, "SSH Keys copied on  " + str(payload["id"]) +
                                    ", try " + str(tries))
                 if os.path.exists(source_file):
                     os.remove(source_file)
                 success = True
             else:
                 self._smgr_log.log(self._smgr_log.ERROR, "Server Matching Server Id:  " + str(payload["id"]) +
                                    " not found. SSH Keys will not be copied. ")
                 tries = 10
                 if os.path.exists(source_file):
                     os.remove(source_file)
                 if sshclient:
                     sshclient.close()
                 success = False
         except Exception as e:
             if os.path.exists(source_file):
                 os.remove(source_file)
             if sshclient:
                 sshclient.close()
             gevent.sleep(30)
     if tries >= 10 and success is False:
         self._smgr_log.log(self._smgr_log.ERROR, "SSH Key copy failed on  " + str(payload["id"]))
     if success and self._provision_immediately_after_reimage == True:
         gevent.spawn(self.gevent_puppet_agent_action, server, self._serverDb, sm_args, "start")
     if success and self.inventory_config_set:
         try:
             url = "http://%s:%s/run_inventory" % (ip, port)
             payload = json.dumps(payload)
             headers = {'content-type': 'application/json'}
             resp = requests.post(url, headers=headers, timeout=5, data=payload)
             return resp.text
         except Exception as e:
             self._smgr_log.log("error", "Error running inventory on  " + str(payload) + " : " + str(e))
             return None
Example #15
0
    def create_store_copy_ssh_keys(self, server_id, server_ip):

        # Create the Keys using Pycrypto
        self._smgr_log.log(self._smgr_log.DEBUG, "Copying keys for server: " + str(server_id))
        ssh_key = paramiko.RSAKey.generate(bits=2048)
        ssh_private_key_obj = StringIO.StringIO()
        ssh_key.write_private_key(ssh_private_key_obj)
        source_file = ""
        bytes_sent = 0
        try:
            subprocess.call(['mkdir', '-p', '/tmp'])
            # Save Public key on Target Server
            source_file = "/tmp/" + str(server_id) + ".pub"
            with open("/tmp/" + str(server_id) + ".pub", 'w+') as content_file:
                content_file.write("ssh-rsa " + str(ssh_key.get_base64()))
                content_file.close()
            ssh = ServerMgrSSHClient(self._serverDb)
            ssh.connect(server_ip, server_id, option="password")
            key_dest_file = "/root/" + str(server_id) + ".pub"
            dest_file = "/root/.ssh/authorized_keys"
            ssh.exec_command("mkdir -p /root/.ssh/")
            ssh.exec_command("touch " + str(key_dest_file))
            if os.path.exists(source_file):
                bytes_sent = ssh.copy(source_file, key_dest_file)
                ssh.exec_command("echo '' >> " + str(dest_file))
                ssh.exec_command("cat " + str(key_dest_file) + " >> " + str(dest_file))
                ssh.exec_command("echo '' >> " + str(dest_file))
                ssh.exec_command("rm -rf " + str(key_dest_file))
            # Update Server table with ssh public and private keys
            update = {'id': server_id,
                      'ssh_public_key': "ssh-rsa " + str(ssh_key.get_base64()),
                      'ssh_private_key': ssh_private_key_obj.getvalue()}
            self._serverDb.modify_server(update)
            ssh.close()
            if os.path.exists(source_file):
                os.remove(source_file)
            self._smgr_log.log(self._smgr_log.DEBUG, "Bytes copied in ssh_key copy: " + str(bytes_sent))
            return ssh_key
        except Exception as e:
            self._smgr_log.log(self._smgr_log.ERROR, "Error Creating/Copying Keys: " + str(server_id) + " : " + str(e))
            if os.path.exists(source_file):
                os.remove(source_file)
            # Update Server table with ssh public and private keys
            update = {'id': server_id,
                      'ssh_public_key': "ssh-rsa " + str(ssh_key.get_base64()),
                      'ssh_private_key': ssh_private_key_obj.getvalue()}
            self._serverDb.modify_server(update)
            return None