def gevent_runner_func(self, hostname, ipmi, ip, username, password, supported_sensors, ipmi_state, sel_event_log_list, option="key"): return_dict = dict() #self.log("info", "Gevent Thread created for %s" % hostname) try: sshclient = ServerMgrSSHClient(serverdb=self._serverDb) sshclient.connect(ip, hostname, option) self.fetch_and_process_resource_info(hostname, ip, sshclient) self.fetch_and_process_network_info(hostname, ip, sshclient) self.fetch_and_process_disk_info(hostname, ip, sshclient) self.fetch_and_process_file_system_view(hostname, ip, sshclient) sshclient.close() except Exception as e: self.log("error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e)) sshclient.close() pass try: return_dict["ipmi_status"] = \ self.fetch_and_process_monitoring(hostname, ipmi, ip, username, password, supported_sensors) self.fetch_and_process_chassis(hostname, ipmi, ip, username, password) if sel_event_log_list: return_dict["sel_log"] = \ self.fetch_and_process_sel_logs(hostname, ipmi, username, password, sel_event_log_list) else: return_dict["sel_log"] = self.fetch_and_process_sel_logs(hostname, ipmi, username, password, []) if not ipmi_state and return_dict["ipmi_status"]: # Trigger REST API CALL to inventory for Server Hostname payload = dict() payload["id"] = str(hostname) self.send_run_inventory_request(self.smgr_ip, self.smgr_port, payload=payload) # self.log("info", "Payload return dict:\n" + str(return_dict)) return return_dict except Exception as e: self.log("error", "Error in getting monitoring info through IPMI for server id: " + str(hostname) + " Error : " + str(e)) pass
def testDiskInfo(self): global package_installed sshclient = ServerMgrSSHClient(None) sshclient.filename = 'disk_info/disk_info_expected_output.txt' self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient) #Test the case where the diff between the previous and current iteration is sent sshclient.filename = 'disk_info/disk_info_diff_expected_output.txt' self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient) #Test the situation where the sysstat package is not installed on the target node package_installed = False self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient) sshclient.filename = 'disk_info/disk_info_wrong_output.txt' #Make sure the exception is raised. If the exception is not raised the test case will fail package_installed = True with self.assertRaises(Exception): self.ipmonitor.fetch_and_process_disk_info("dummy","dummy",sshclient)
def testDiskInfo(self): global package_installed sshclient = ServerMgrSSHClient(None) sshclient.filename = 'disk_info/disk_info_expected_output.txt' self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient) #Test the case where the diff between the previous and current iteration is sent sshclient.filename = 'disk_info/disk_info_diff_expected_output.txt' self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient) #Test the situation where the sysstat package is not installed on the target node package_installed = False self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient) sshclient.filename = 'disk_info/disk_info_wrong_output.txt' #Make sure the exception is raised. If the exception is not raised the test case will fail package_installed = True with self.assertRaises(Exception): self.ipmonitor.fetch_and_process_disk_info("dummy", "dummy", sshclient)
def gevent_runner_function(self, action, hostname, ip=None, ipmi=None, username=None, password=None, option="key"): sshclient = None if action == "add" and ip and ipmi and username and password: try: sshclient = ServerMgrSSHClient(serverdb=self._serverDb) sshclient.connect(ip, hostname, option) self.get_facter_info(hostname, ip, sshclient) self.get_cpu_info(hostname, ip, sshclient) self.get_ethernet_info(hostname, ip, sshclient) self.get_memory_info(hostname, ip, sshclient) sshclient.close() except Exception as e: if sshclient: sshclient.close() self.log( "error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e)) pass try: self.get_fru_info(hostname, ipmi, username, password) except Exception as e: self.log( "error", "Error in getting Inventory Info through IPMI for server id: " + str(hostname) + " Error : " + str(e)) pass elif action == "delete": self.log(self.INFO, "Deleted info of server: %s" % hostname) self.delete_inventory_info(hostname)
def gevent_runner_function(self, action, hostname, ip=None, ipmi=None, username=None, password=None, option="key"): sshclient = None try: if action == "add" and ip and ipmi and username and password: sshclient = ServerMgrSSHClient(serverdb=self._serverDb) sshclient.connect(ip, hostname, option) self.get_fru_info(hostname, ipmi, username, password) self.get_facter_info(hostname, ip, sshclient) self.get_cpu_info(hostname, ip, sshclient) self.get_ethernet_info(hostname, ip, sshclient) self.get_memory_info(hostname, ip, sshclient) sshclient.close() elif action == "delete": self.log(self.INFO, "Deleted info of server: %s" % hostname) self.delete_inventory_info(hostname) except Exception as e: if sshclient: sshclient.close() self.log("error", "Gevent SSH Connect Execption for server id: " + str(hostname) + " Error : " + str(e)) pass
def gevent_puppet_agent_action(self,server, serverDb, sm_args, action,access_method="key"): success = False tries = 0 gevent.sleep(30) self._smgr_log.log("debug", "Going to %s the puppet agent on the server %s"% (action, str(server['id']))) while not success and tries < int(sm_args.puppet_agent_retry_count): try: tries += 1 sshclient = ServerMgrSSHClient(serverdb=serverDb) sshclient.connect(str(server['ip_address']), str(server['id']), access_method) op = sshclient.exec_command('python -c "import platform; print platform.linux_distribution()"') self._smgr_log.log("debug", "OP is %s" %op) os_type = 'centos' if 'centos' in op.lower() else 'ubuntu' if os_type == 'centos': enable_puppet_svc_cmd = "chkconfig puppet on" disable_puppet_svc_cmd = "chkconfig puppet off" else: enable_puppet_svc_cmd = " sed -i 's/START=.*$/START=yes/' /etc/default/puppet && "\ "/usr/bin/puppet resource service puppet ensure=running enable=true " disable_puppet_svc_cmd = " sed -i 's/START=.*$/START=no/' /etc/default/puppet && " \ "/usr/bin/puppet resource service puppet ensure=stopped enable=false " self._smgr_log.log("debug", "PUPPET START Command is %s" %enable_puppet_svc_cmd) if action == "start": output = sshclient.exec_command(enable_puppet_svc_cmd) self._smgr_log.log("debug", "OUTPUT1 is %s" %output) output = sshclient.exec_command("puppet agent --enable") self._smgr_log.log("debug", "OUTPUT2 is %s" %output) output = sshclient.exec_command("service puppet restart") self._smgr_log.log("debug", "OUTPUT3 is %s" %output) self._smgr_log.log("debug", "Successfully started the puppet agent on the server " + str(server['id'])) self._provision_immediately_after_reimage = False else: output = sshclient.exec_command(disable_puppet_svc_cmd) output = sshclient.exec_command("puppet agent --disable") output = sshclient.exec_command("service puppet stop") self._smgr_log.log("debug", "Successfully stopped the puppet agent on the server " + str(server['id'])) success = True sshclient.close() except Exception as e: if action == "start": servers = self._serverDb.get_server({"id": server['id']}, detail=True) server_state = servers[0]['status'] if server_state == "reimage_started" or server_state == "restart_issued" \ or server_state == "reimage_completed" or server_state == "provision_issued": self._provision_immediately_after_reimage = True if sshclient: sshclient.close() self._smgr_log.log(self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e)) self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" %(action, str(server["id"]), str(tries))) gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds)) if tries >= int(sm_args.puppet_agent_retry_count) and success is False: if action == "start": self._smgr_log.log(self._smgr_log.ERROR, "Starting the puppet agent failed on " + str(server["id"])) else: self._smgr_log.log(self._smgr_log.ERROR, "Stopping the puppet agent failed on " + str(server["id"]))
def copy_ssh_keys_to_server(self, ip_address, server_id): self._smgr_log.log(self._smgr_log.DEBUG, "COPY-KEY: Server: " + str(server_id)) tries = 0 gevent.sleep(60) success = False while True: try: tries = tries + 1 #avoid overflow if tries > 10000: tries = 10000 # We keep trying infinitely for this, so if server is deleted before # this, we need to know about it and pull new_public_key servers = self._serverDb.get_server({"id": server_id}, detail=True) if not servers : self._smgr_log.log(self._smgr_log.DEBUG, "COPY-KEY: Server: " + str(server_id) + " NOT FOUND") return False server = servers[0] ssh_key = server["ssh_public_key"] #try to connect first if target node is up source_file = "/tmp/" + str(server_id) + ".pub" ssh = ServerMgrSSHClient(self._serverDb) ssh.connect(ip_address, server_id, option="password") subprocess.call(['mkdir', '-p', '/tmp']) with open("/tmp/" + str(server_id) + ".pub", 'w+') as content_file: content_file.write(str(ssh_key)) content_file.close() key_dest_file = "/root/" + str(server_id) + ".pub" dest_file = "/root/.ssh/authorized_keys" ssh.exec_command("mkdir -p /root/.ssh/") ssh.exec_command("touch " + str(key_dest_file)) ssh.exec_command("touch " + str(dest_file)) if os.path.exists(source_file): # Copy Public key on Target Server #TODO: check if authrized keys are already available bytes_sent = ssh.copy(source_file, key_dest_file) cmd = "grep -q -f " + str(key_dest_file) + " " + str(dest_file) + ";" \ + " RETVAL=$? ; " \ + " if [[ $RETVAL -eq 1 ]]; then \ echo '' >> " + str(dest_file) + "; \ cat " + str(key_dest_file) + " >> " + str(dest_file) + ";\ echo '' >> " + str(dest_file) + "; fi; \ rm -f " + str(key_dest_file) + "; " self._smgr_log.log(self._smgr_log.DEBUG, cmd) ssh.exec_command(cmd) ssh.close() if os.path.exists(source_file): os.remove(source_file) msg = "COPY-KEYS: %s bytes copied on %s: " %(str(bytes_sent), str(server_id)) self._smgr_log.log(self._smgr_log.DEBUG, msg) success = True return success except Exception as e: msg = "COPY-KEYS: Host : %s Try: %d: ERROR Copying Keys: %s" % (str(server_id), tries, str(e)) self._smgr_log.log(self._smgr_log.ERROR, msg) if os.path.exists(source_file): os.remove(source_file) if ssh: ssh.close() if tries >= 20: sleep_time = 120 else: sleep_time = 30 gevent.sleep(sleep_time)
def reimage_run_inventory(self, ip, port, payload): success = False source_file = "" sshclient = ServerMgrSSHClient(serverdb=self._serverDb) self._smgr_log.log("debug", "Running Reimage Inventory on " + str(payload)) if not self.inventory_config_set and not self.monitoring_config_set: return "No Inventory or Monitoring Configured" tries = 0 gevent.sleep(60) while not success and tries < 10: try: tries += 1 server = self._serverDb.get_server({"id": str(payload["id"])}, detail=True) if server and len(server) == 1: server = server[0] subprocess.call(['ssh-keygen', '-f', '/root/.ssh/known_hosts', '-R', str(server["ip_address"])]) subprocess.call(['mkdir', '-p', '/tmp']) sshclient.connect(str(server["ip_address"]), str(server["id"]), "password") match_dict = dict() match_dict["id"] = str(payload["id"]) self._smgr_log.log(self._smgr_log.DEBUG, "Running inventory on " + str(payload["id"]) + ", try " + str(tries)) ssh_public_ket_str = str(server["ssh_public_key"]) with open("/tmp/" + str(payload["id"]) + ".pub", 'w+') as content_file: content_file.write(ssh_public_ket_str) content_file.close() source_file = "/tmp/" + str(payload["id"]) + ".pub" dest_file = "/root/.ssh/authorized_keys" if os.path.exists(source_file): sshclient.exec_command('mkdir -p /root/.ssh/') sshclient.exec_command('touch /root/.ssh/authorized_keys') bytes_sent = sshclient.copy(source_file, dest_file) sshclient.exec_command("echo '' >> " + str(dest_file)) sshclient.close() self._smgr_log.log(self._smgr_log.DEBUG, "SSH Keys copied on " + str(payload["id"]) + ", try " + str(tries)) if os.path.exists(source_file): os.remove(source_file) success = True else: self._smgr_log.log(self._smgr_log.ERROR, "Server Matching Server Id: " + str(payload["id"]) + " not found. SSH Keys will not be copied. ") tries = 10 if os.path.exists(source_file): os.remove(source_file) if sshclient: sshclient.close() success = False except Exception as e: if os.path.exists(source_file): os.remove(source_file) if sshclient: sshclient.close() gevent.sleep(30) if tries >= 10 and success is False: self._smgr_log.log(self._smgr_log.ERROR, "SSH Key copy failed on " + str(payload["id"])) if success and self.inventory_config_set: try: url = "http://%s:%s/run_inventory" % (ip, port) payload = json.dumps(payload) headers = {'content-type': 'application/json'} resp = requests.post(url, headers=headers, timeout=5, data=payload) return resp.text except Exception as e: self._smgr_log.log("error", "Error running inventory on " + str(payload) + " : " + str(e)) return None
def create_store_copy_ssh_keys(self, server_id, server_ip): # Create the Keys using Pycrypto self._smgr_log.log(self._smgr_log.DEBUG, "Copying keys for server: " + str(server_id)) ssh_key = paramiko.RSAKey.generate(bits=2048) ssh_private_key_obj = StringIO.StringIO() ssh_key.write_private_key(ssh_private_key_obj) source_file = "" bytes_sent = 0 try: subprocess.call(['mkdir', '-p', '/tmp']) # Save Public key on Target Server source_file = "/tmp/" + str(server_id) + ".pub" with open("/tmp/" + str(server_id) + ".pub", 'w+') as content_file: content_file.write("ssh-rsa " + str(ssh_key.get_base64())) content_file.close() ssh = ServerMgrSSHClient(self._serverDb) ssh.connect(server_ip, server_id, option="password") key_dest_file = "/root/" + str(server_id) + ".pub" dest_file = "/root/.ssh/authorized_keys" ssh.exec_command("mkdir -p /root/.ssh/") ssh.exec_command("touch " + str(key_dest_file)) if os.path.exists(source_file): bytes_sent = ssh.copy(source_file, key_dest_file) ssh.exec_command("echo '' >> " + str(dest_file)) ssh.exec_command("cat " + str(key_dest_file) + " >> " + str(dest_file)) ssh.exec_command("echo '' >> " + str(dest_file)) ssh.exec_command("rm -rf " + str(key_dest_file)) # Update Server table with ssh public and private keys update = {'id': server_id, 'ssh_public_key': "ssh-rsa " + str(ssh_key.get_base64()), 'ssh_private_key': ssh_private_key_obj.getvalue()} self._serverDb.modify_server(update) ssh.close() if os.path.exists(source_file): os.remove(source_file) self._smgr_log.log(self._smgr_log.DEBUG, "Bytes copied in ssh_key copy: " + str(bytes_sent)) return ssh_key except Exception as e: self._smgr_log.log(self._smgr_log.ERROR, "Error Creating/Copying Keys: " + str(server_id) + " : " + str(e)) if os.path.exists(source_file): os.remove(source_file) # Update Server table with ssh public and private keys update = {'id': server_id, 'ssh_public_key': "ssh-rsa " + str(ssh_key.get_base64()), 'ssh_private_key': ssh_private_key_obj.getvalue()} self._serverDb.modify_server(update) return None
def gevent_puppet_agent_action(self,server, serverDb, sm_args, action,access_method="key"): success = False tries = 0 gevent.sleep(30) self._smgr_log.log("debug", "Going to %s the puppet agent on the server %s"% (action, str(server['id']))) while not success and tries < int(sm_args.puppet_agent_retry_count): try: tries += 1 sshclient = ServerMgrSSHClient(serverdb=serverDb) sshclient.connect(str(server['ip_address']), str(server['id']), access_method) if action == "start": output = sshclient.exec_command("sed -i 's/START=.*$/START=yes/' /etc/default/puppet") output = sshclient.exec_command("service puppet start") self._smgr_log.log("debug", "Successfully started the puppet agent on the server " + str(server['id'])) self._provision_immediately_after_reimage = False else: output = sshclient.exec_command("sed -i 's/START=.*$/START=no/' /etc/default/puppet") output = sshclient.exec_command("service puppet stop") self._smgr_log.log("debug", "Successfully stopped the puppet agent on the server " + str(server['id'])) success = True sshclient.close() except Exception as e: if action == "start": server_state = server['status'] if server_state == "reimage_started" or server_state == "restart_issued" or server_state == "reimage_completed": self._provision_immediately_after_reimage = True if sshclient: sshclient.close() self._smgr_log.log(self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e)) self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" %(action, str(server["id"]), str(tries))) gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds)) if tries >= int(sm_args.puppet_agent_retry_count) and success is False: if action == "start": self._smgr_log.log(self._smgr_log.ERROR, "Starting the puppet agent failed on " + str(server["id"])) else: self._smgr_log.log(self._smgr_log.ERROR, "Stopping the puppet agent failed on " + str(server["id"]))
def gevent_puppet_agent_action(self, server, serverDb, sm_args, action, access_method="key"): success = False tries = 0 gevent.sleep(30) self._smgr_log.log( "debug", "Going to %s the puppet agent on the server %s" % (action, str(server['id']))) while not success and tries < int(sm_args.puppet_agent_retry_count): try: tries += 1 sshclient = ServerMgrSSHClient(serverdb=serverDb) sshclient.connect(str(server['ip_address']), str(server['id']), access_method) op = sshclient.exec_command( 'python -c "import platform; print platform.linux_distribution()"' ) self._smgr_log.log("debug", "OP is %s" % op) os_type = 'centos' if 'centos' in op.lower() else 'ubuntu' if os_type == 'centos': enable_puppet_svc_cmd = "chkconfig puppet on" disable_puppet_svc_cmd = "chkconfig puppet off" else: enable_puppet_svc_cmd = " sed -i 's/START=.*$/START=yes/' /etc/default/puppet && "\ "/usr/bin/puppet resource service puppet ensure=running enable=true " disable_puppet_svc_cmd = " sed -i 's/START=.*$/START=no/' /etc/default/puppet && " \ "/usr/bin/puppet resource service puppet ensure=stopped enable=false " self._smgr_log.log( "debug", "PUPPET START Command is %s" % enable_puppet_svc_cmd) if action == "start": output = sshclient.exec_command(enable_puppet_svc_cmd) self._smgr_log.log("debug", "OUTPUT1 is %s" % output) output = sshclient.exec_command("puppet agent --enable") self._smgr_log.log("debug", "OUTPUT2 is %s" % output) output = sshclient.exec_command("service puppet restart") self._smgr_log.log("debug", "OUTPUT3 is %s" % output) self._smgr_log.log( "debug", "Successfully started the puppet agent on the server " + str(server['id'])) self._provision_immediately_after_reimage = False else: output = sshclient.exec_command(disable_puppet_svc_cmd) output = sshclient.exec_command("puppet agent --disable") output = sshclient.exec_command("service puppet stop") self._smgr_log.log( "debug", "Successfully stopped the puppet agent on the server " + str(server['id'])) success = True sshclient.close() except Exception as e: if action == "start": servers = self._serverDb.get_server({"id": server['id']}, detail=True) server_state = servers[0]['status'] if server_state == "reimage_started" or server_state == "restart_issued" \ or server_state == "reimage_completed" or server_state == "provision_issued": self._provision_immediately_after_reimage = True if sshclient: sshclient.close() self._smgr_log.log( self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e)) self._smgr_log.log( self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" % (action, str(server["id"]), str(tries))) gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds)) if tries >= int(sm_args.puppet_agent_retry_count) and success is False: if action == "start": self._smgr_log.log( self._smgr_log.ERROR, "Starting the puppet agent failed on " + str(server["id"])) else: self._smgr_log.log( self._smgr_log.ERROR, "Stopping the puppet agent failed on " + str(server["id"]))
def copy_ssh_keys_to_server(self, ip_address, server_id): self._smgr_log.log(self._smgr_log.DEBUG, "COPY-KEY: Server: " + str(server_id)) tries = 0 gevent.sleep(60) success = False while True: try: tries = tries + 1 #avoid overflow if tries > 10000: tries = 10000 # We keep trying infinitely for this, so if server is deleted before # this, we need to know about it and pull new_public_key servers = self._serverDb.get_server({"id": server_id}, detail=True) if not servers: self._smgr_log.log( self._smgr_log.DEBUG, "COPY-KEY: Server: " + str(server_id) + " NOT FOUND") return False server = servers[0] ssh_key = server["ssh_public_key"] #try to connect first if target node is up source_file = "/tmp/" + str(server_id) + ".pub" ssh = ServerMgrSSHClient(self._serverDb) ssh.connect(ip_address, server_id, option="password") subprocess.call(['mkdir', '-p', '/tmp']) with open("/tmp/" + str(server_id) + ".pub", 'w+') as content_file: content_file.write(str(ssh_key)) content_file.close() key_dest_file = "/root/" + str(server_id) + ".pub" dest_file = "/root/.ssh/authorized_keys" ssh.exec_command("mkdir -p /root/.ssh/") ssh.exec_command("touch " + str(key_dest_file)) ssh.exec_command("touch " + str(dest_file)) if os.path.exists(source_file): # Copy Public key on Target Server #TODO: check if authrized keys are already available bytes_sent = ssh.copy(source_file, key_dest_file) cmd = "grep -q -f " + str(key_dest_file) + " " + str(dest_file) + ";" \ + " RETVAL=$? ; " \ + " if [[ $RETVAL -eq 1 ]]; then \ echo '' >> " + str(dest_file) + "; \ cat " + str(key_dest_file) + " >> " + str(dest_file) + ";\ echo '' >> " + str(dest_file) + "; fi; \ rm -f " + str(key_dest_file) + "; " self._smgr_log.log(self._smgr_log.DEBUG, cmd) ssh.exec_command(cmd) ssh.close() if os.path.exists(source_file): os.remove(source_file) msg = "COPY-KEYS: %s bytes copied on %s: " % (str(bytes_sent), str(server_id)) self._smgr_log.log(self._smgr_log.DEBUG, msg) success = True return success except Exception as e: msg = "COPY-KEYS: Host : %s Try: %d: ERROR Copying Keys: %s" % ( str(server_id), tries, str(e)) self._smgr_log.log(self._smgr_log.ERROR, msg) if os.path.exists(source_file): os.remove(source_file) if ssh: ssh.close() if tries >= 20: sleep_time = 120 else: sleep_time = 30 gevent.sleep(sleep_time)
def gevent_puppet_agent_action(self,server, serverDb, sm_args, action,access_method="key"): success = False tries = 0 gevent.sleep(30) pdb.set_trace() self._smgr_log.log("debug", "Going to %s the puppet agent on the server %s"% (action, str(server['id']))) while not success and tries < int(sm_args.puppet_agent_retry_count): try: tries += 1 sshclient = ServerMgrSSHClient(serverdb=serverDb) sshclient.connect(str(server['ip_address']), str(server['id']), access_method) if action == "start": output = sshclient.exec_command("sed -i 's/START=.*$/START=yes/' /etc/default/puppet") output = sshclient.exec_command("service puppet start") self._smgr_log.log("debug", "Successfully started the puppet agent on the server " + str(server['id'])) self._provision_immediately_after_reimage = False else: output = sshclient.exec_command("sed -i 's/START=.*$/START=no/' /etc/default/puppet") output = sshclient.exec_command("service puppet stop") self._smgr_log.log("debug", "Successfully stopped the puppet agent on the server " + str(server['id'])) success = True sshclient.close() except Exception as e: if action == "start": server_state = server['status'] if server_state == "reimage_started" or server_state == "restart_issued" or server_state == "reimage_completed": self._provision_immediately_after_reimage = True if sshclient: sshclient.close() self._smgr_log.log(self._smgr_log.ERROR, "Gevent SSH Connect Exception for server id: " + server['id'] + " Error : " + str(e)) self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to %s the puppet agent in the server %s, try %s" %(action, str(server["id"]), str(tries))) gevent.sleep(int(sm_args.puppet_agent_retry_poll_interval_seconds)) if tries >= int(sm_args.puppet_agent_retry_count) and success is False: if action == "start": self._smgr_log.log(self._smgr_log.ERROR, "Starting the puppet agent failed on " + str(server["id"])) else: self._smgr_log.log(self._smgr_log.ERROR, "Stopping the puppet agent failed on " + str(server["id"]))
def copy_ssh_keys_to_servers(self, ip, port, payload, sm_args=None): success = False source_file = "" sshclient = ServerMgrSSHClient(serverdb=self._serverDb) self._smgr_log.log("debug", "Copying the ssh keys " + str(payload)) tries = 0 gevent.sleep(60) while not success and tries < 10: try: tries += 1 server = self._serverDb.get_server({"id": str(payload["id"])}, detail=True) if server and len(server) == 1: server = server[0] subprocess.call(['ssh-keygen', '-f', '/root/.ssh/known_hosts', '-R', str(server["ip_address"])]) subprocess.call(['mkdir', '-p', '/tmp']) sshclient.connect(str(server["ip_address"]), str(server["id"]), "password") match_dict = dict() match_dict["id"] = str(payload["id"]) self._smgr_log.log(self._smgr_log.DEBUG, "Still trying to copy the ssh keys " + str(payload["id"]) + ", try " + str(tries)) ssh_public_ket_str = str(server["ssh_public_key"]) with open("/tmp/" + str(payload["id"]) + ".pub", 'w+') as content_file: content_file.write(ssh_public_ket_str) content_file.close() source_file = "/tmp/" + str(payload["id"]) + ".pub" dest_file = "/root/.ssh/authorized_keys" if os.path.exists(source_file): sshclient.exec_command('mkdir -p /root/.ssh/') sshclient.exec_command('touch /root/.ssh/authorized_keys') bytes_sent = sshclient.copy(source_file, dest_file) sshclient.exec_command("echo '' >> " + str(dest_file)) sshclient.close() self._smgr_log.log(self._smgr_log.DEBUG, "SSH Keys copied on " + str(payload["id"]) + ", try " + str(tries)) if os.path.exists(source_file): os.remove(source_file) success = True else: self._smgr_log.log(self._smgr_log.ERROR, "Server Matching Server Id: " + str(payload["id"]) + " not found. SSH Keys will not be copied. ") tries = 10 if os.path.exists(source_file): os.remove(source_file) if sshclient: sshclient.close() success = False except Exception as e: if os.path.exists(source_file): os.remove(source_file) if sshclient: sshclient.close() gevent.sleep(30) if tries >= 10 and success is False: self._smgr_log.log(self._smgr_log.ERROR, "SSH Key copy failed on " + str(payload["id"])) if success and self._provision_immediately_after_reimage == True: gevent.spawn(self.gevent_puppet_agent_action, server, self._serverDb, sm_args, "start") if success and self.inventory_config_set: try: url = "http://%s:%s/run_inventory" % (ip, port) payload = json.dumps(payload) headers = {'content-type': 'application/json'} resp = requests.post(url, headers=headers, timeout=5, data=payload) return resp.text except Exception as e: self._smgr_log.log("error", "Error running inventory on " + str(payload) + " : " + str(e)) return None