def __init__(self, cam_num, ssh): self.cam_num = cam_num self.last_frame = np.zeros((1,1)) self.ssh = ssh self.scp = SCPClient(self.ssh.get_transport(), socket_timeout=None)
def _startRemoteJob(self): try: cmd = self._getMpiCommand() remoteConfig = self.jobParams[PARAM_REMOTE_JOB_CONFIG] remoteWorkingDir = remoteConfig[u"repo"] remoteHost = remoteConfig[u"host"] remoteUser = remoteConfig[u"user"] remotePass = remoteConfig[u"pass"] if self.jobParams[PARAM_SIM_MAX_RUNNING_TIME] > 0: maxRuntime = datetime.timedelta( minutes=self.jobParams[PARAM_SIM_MAX_RUNNING_TIME]) else: maxRuntime = u"99-00:00:00" template = remoteConfig[u"template"] template = template.replace(u"{PARAM_SIM_MAX_RUNNING_TIME}", unicode(maxRuntime)) template = template.replace( u"{PARAM_SIM_NODES}", unicode(self.jobParams[PARAM_SIM_NODES])) template = template.replace(u"{PARAM_REMOTE_WORKING_DIR}", unicode(remoteWorkingDir)) template = template.replace(u"{PARAM_JOB_COMMAND}", cmd) self.logger.debug(u"Remote job started") fd, path = tempfile.mkstemp() with os.fdopen(fd, 'w') as tmp: # do stuff with temp file tmp.write(template) from paramiko import SSHClient from scp import SCPClient ssh = SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(hostname=remoteHost, username=remoteUser, password=remotePass) with SCPClient(ssh.get_transport()) as scp: scp.put(path, '{0}/remoteSbatchJob.sh'.format(remoteWorkingDir)) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( 'sbatch {0}/remoteSbatchJob.sh'.format(remoteWorkingDir)) self.remoteJobId = int( re.search("[0-9]*$", ssh_stdout.readlines()[0]).group(0)) while True: ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( 'scontrol show jobid -dd {0}'.format(self.remoteJobId)) line = ssh_stdout.readlines()[3] jobStatus = re.search(u"JobState=(\w*)", line).group(1) if jobStatus == "PENDING": self.status = STATUS_WAITING elif jobStatus == "RUNNING": self.status = STATUS_RUNNING elif jobStatus == "CANCELLED": self.status = STATUS_ERROR break elif jobStatus == "ERROR": self.status = STATUS_ERROR break elif jobStatus == "COMPLETED": self.status = STATUS_FINISHED scp.get( '{0}/{1}'.format(remoteWorkingDir, self.outputFile), self.outputFile) break self.sigJobStatusChanged.emit(self.status) time.sleep(1) self.sigJobStatusChanged.emit(self.status) except Exception as e: self.logger.error(u"Exception: {0}".format(e)) self.status = STATUS_ERROR self.sigJobStatusChanged.emit(self.status) finally: self.sigFinishJob.emit()
def copy_to_host(host, remote_file, local_file, is_root=False): """Copy file to remote host Function first copies file to remote user directory and then moving. Final move depends to right to directory. """ _lpath, lfilename = os.path.split(local_file) rpath, rfilename = os.path.split(remote_file) local_filepath = os.path.join('./', lfilename) if len(rfilename) < 1: rfilename = lfilename remote_file = os.path.join(rpath, rfilename) if not os.path.isfile(local_file): logger.error("File to copy does not exist", file=local_file, module=COMMAND_MODULE_CUSTOM) return False ssh_run = SshRun(host.hostname, host.ssh_hostname, local_file, helpers.get_function_name(), is_root) if not ssh_run.create_ssh_channel(): return False success = True if not CREDS.DRY_RUN: retries = 0 success = False while retries < MAX_SCP_RETRIES: # Copies file to remote users directory with SCPClient(ssh_run.ssh.get_transport()) as scp: try: scp.put(local_file, lfilename) success = True break except Exception as e: if _error_check(e.message, remote_file, host.hostname, "copy_to_host"): return retries += 1 time.sleep(2) if not success: logger.error("Problem using scp", hostname=host.hostname, local_file=local_file, module=COMMAND_MODULE_CUSTOM) return False if local_filepath != remote_file: # Only copy file if not the same location ssh_run.add_cmd("mkdir -p %s" % rpath) ssh_run.add_cmd("mv %s %s" % (lfilename, remote_file)) if is_root: # Make sure root permission are set if needed ssh_run.add_cmd("chown root:root %s" % remote_file) ssh_run.add_cmd("restorecon %s" % remote_file) results = ssh_run.run() _log_rc(results, ssh_run.function_name, hostname=host.hostname, remote_file=remote_file, local_file=local_file, outputs=results['outputs'], module=COMMAND_MODULE_BUILTIN) return results['rc'] < 1
import paramiko from scp import SCPClient ssh_client = paramiko.SSHClient() ssh_client.load_system_host_keys() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_client.connect('10.1.10.200', port=22, username='******', password='******', look_for_keys=False, allow_agent=False) scp = SCPClient(ssh_client.get_transport()) #scp.put('10.1.10.200-2019-10-9.txt', '/var/tmp/newfile.txt') #Copy a directory #scp.put('directory1', recursive=True, remote_path='/var/tmp') scp.get('/var/tmp/directory1/10.1.10.200-2019-10-9.txt', 'copiedfile.txt') scp.close()
def Update(sip, S_md5, App, path): ssh = paramiko.SSHClient() key = paramiko.RSAKey.from_private_key_file(key_file) ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) for ip in sip: ip = ip.strip() Redis.lpush(Key, '-' * 60) Redis.lpush(Key, 'publish ' + path + ' to ' + ip + '\n') try: ssh.connect(ip, 22, username, pkey=key, timeout=15) cmd = "mkdir -p {0}".format(web_path) ssh.exec_command(cmd) scp = SCPClient(ssh.get_transport()) scp.put('%s%s.zip' % (web_path, App), web_path) cmd = '/usr/bin/md5sum %s/%s.zip' % (web_path, App) stdin, stdout, stderr = ssh.exec_command(cmd) R_md5 = stdout.read().split()[0] if R_md5 == S_md5: Redis.lpush(Key, '%s md5 verify --->pass!' % App) Redis.lpush(Key, 'unzip start ......') cmds = [ 'cd %s && /usr/bin/unzip -qo %s.zip && /bin/rm -f %s.zip' % (web_path, App, App), '[ -e %s ] && echo ok' % tag_path ] for cmd in cmds: stdin, stdout, stderr = ssh.exec_command(cmd) result_zip = stdout.read().strip() if result_zip == 'ok': Redis.lpush(Key, '%s --->unzip success!' % App) Redis.lpush(Key, 'backup start ......') cmds = ('/bin/mkdir -p %s' % bak_path, '[ -e %s/%s ] && /bin/rm -rf %s/%s' % (bak_path, App, bak_path, App), '[ -e %s%s ] && /bin/mv %s%s %s/%s' % (web_path, App, web_path, App, bak_path, App)) for cmd in cmds: stdin, stdout, stderr = ssh.exec_command(cmd) result_backup = stderr.read() if result_backup: Redis.lpush(Key, result_backup) Redis.lpush(Key, '%s --->backup fail!' % App) else: Redis.lpush(Key, '%s --->backup success!' % App) Redis.lpush(Key, 'publish start ......') cmd = '/bin/mv %s %s%s' % (tag_path, web_path, App) stdin, stdout, stderr = ssh.exec_command(cmd) result_rsync = stderr.read() if result_rsync: Redis.lpush(Key, result_rsync) Redis.lpush(Key, '%s --->publish fail!' % App) else: Redis.lpush( Key, '%s --->publish success!' % App) ssh.close() Redis.lpush(Key, '-' * 60) else: Redis.lpush(Key, '%s --->unzip fail!' % App) else: Redis.lpush(Key, 'S_MD5:%r' % S_md5) Redis.lpush(Key, 'R_MD5:%r' % R_md5) Redis.lpush(Key, ' verify %s on %s fail!!!' % (App, ip)) except Exception as e: Redis.lpush(Key, 'Update:{0}'.format(e)) ssh.close() sys.exit()
def __init__(self, file_path, timer, onehot=False, reward_scaler=1e6, op_age=min, save_ll=False, remote=False, with_age=True, ip_address=None, h_size=2): super(LLVMEnv, self).__init__() self.file_path = file_path self.reward_scaler = reward_scaler self.timer = timer self.extremum = op_age self.onehot = onehot self.MAX_AGE = 10 self.save_ll = save_ll self.with_age = with_age self.ssh_con = False self.scp_con = False self.ip_address = None self.h_size = h_size # size of the history # configure ssh connection to raspberryPi if remote: self.ip_address = ip_address self.ssh_con = paramiko.SSHClient() self.ssh_con.load_system_host_keys() self.ssh_con.connect(self.ip_address, username="******", password="******") self.scp_con = SCPClient(self.ssh_con.get_transport()) # instructions handle by the environment self.instr_to_opcode = { "call": 1, "tail": 1, "store": 2, "load": 3, "fadd": 4, "fsub": 5, "fmul": 6, "fdiv": 7, "bitcast": 8, "fpext": 9, "sitofp": 10, "fptrunc": 11, "other": 12 } # self.instr_to_opcode = { # "alloca": 1, # "alloca": 1, # "store": 2, # "load": 3, # "fpext": 4, # "fmul": 5, # "fdiv": 6, # "sitofp": 7, # "other": 8 # } self.max_step = 50000 self.curr_step = 0 # Define action and state spaces # 2 actions: choose instruction or not. self.action_space = spaces.Discrete(2) if not self.onehot: if self.with_age: self.observation_space = spaces.Box( low=np.array([0, 0, 0, 0, 1]), high=np.array([12, 12, 12, self.MAX_AGE, 4]), dtype=int) else: self.observation_space = spaces.Box(low=np.array([0, 0, 0, 0]), high=np.array( [12, 12, 12, 4]), dtype=int) else: if self.with_age: low = np.concatenate( (np.array([0] * (self.h_size + 1) * len(self.instr_to_opcode)), np.array([0] * (self.MAX_AGE + 1)), np.array([0] * 4))) high = np.concatenate( (np.array([1] * (self.h_size + 1) * len(self.instr_to_opcode)), np.array([1] * (self.MAX_AGE + 1)), np.array([1] * 4))) else: low = np.concatenate( (np.array([0] * (self.h_size + 1) * (len(self.instr_to_opcode))), np.array([0] * 4))) high = np.concatenate( (np.array([1] * (self.h_size + 1) * (len(self.instr_to_opcode))), np.array([1] * 4))) self.observation_space = spaces.Box(low=low, high=high, dtype=int) self.selected_instruction = None # current selected instruction self.schedulable_instructions = None # last scheduled instructions's opcode self.history = collections.deque([0] * (self.h_size), self.h_size) self.llvm = LLVMController(self.file_path) print("timing original file") self.original_time = self.llvm.run(self.timer, self.curr_step, False, self.ssh_con, self.scp_con, self.ip_address, time_original=True) print(f"original program time is: {self.original_time}s")
ssh.load_system_host_keys() ssh.connect('sundog.uchicago.edu',username='******') loading_string = '' while True: files = [] for ext in extensions: files.extend(glob.glob(input_dir + ext)) for f in files: f = f.replace('\\','/') # To handle windows paths. t = os.path.getmtime(f) if t >= call_time: loading_string = '.' # SCPCLient takes a paramiko transport and progress callback as its arguments. # Might want to look into a keep alive if I only want to ssh once and not every call. with SCPClient(ssh.get_transport(), progress=progress) as scp: #ready file f_new = output_dir + f.split('/')[-1] try: print('Removing %s'%f) os.remove(f) except Exception as e: print('Failed deleting %s.'%f) print(e) #met file try: f = f.replace('.ready','.met') f_new = f_new.replace('.ready','.met') print('Transferring met file modified since last call') scp.put(f, f_new) print('\n')
def getDataFromRemote(self, sshClient, ori, dest): scp = SCPClient(sshClient.getConnection().get_transport()) scp.get(ori, recursive=True, local_path=dest) scp.close()
def connect_transfer_client(self): print("Connected to server", self.host) self.scp_client = SCPClient(self.client.get_transport(), progress=self.progress)
ssh.set_missing_host_key_policy(AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(hostname, username='******', password='******', timeout=120) for index in range(IMAGES_PER_MACHINE): print("Limpiando foto %d: " % index) ssh.exec_command("rm -f /tmp/out.xwd") ssh.exec_command("rm -f /tmp/img.jpg") print("Tomando Foto %d: " % index) ssh.exec_command("xwd -out /tmp/out.xwd -root -display :0.0 ", timeout=50) print("transformando foto: %d: " % index) ssh.exec_command("convert -quality 8% /tmp/out.xwd /tmp/img.jpg", timeout=50) print("Copiando imagen ... ") fname = "img_%d_%s.jpg" % ( index, str(datetime.now().strftime("%Y%m%d-%H_%M_%S"))) filepath = "%s/%s" % (maquina, fname) try: with SCPClient(ssh.get_transport(), socket_timeout=30) as scp: scp.get('/tmp/img.jpg', filepath) except Exception as e: print("error al copiar la imagen %d" % index, e) ssh.close() except Exception as e: print("error en la conexion", e)
def putDataInRemote(self, sshClient, ori, dest): scp = SCPClient(sshClient.getConnection().get_transport()) scp.put(ori, dest, recursive=True) scp.close()
def install_apache(): # Read apche hosts metadata from the json file with open('hosts.json', 'r') as fp: data = json.load(fp) for hosts in data['apache_hosts']: print hosts name = hosts['cname'] user = hosts['username'] passwd = hosts['password'] diskt = hosts['disk_threshold'] override = hosts['override_port'] install = hosts['install_pkg'] port = hosts['web_port'] documentroot = hosts['documentroot'] logging.debug("%s, %s, %s, %,s", name, user, passwd, install) # Command to install apache related packages ssh = SshConnect() ssh.connection(name, user, passwd) ssh1 = ssh.connection(name, user, passwd) scp = SCPClient(ssh1) scp.put('scripts/find_deleted.sh', '/tmp/find_deleted.sh') output = ssh.run_command('sh -c /tmp/find_deleted.sh') command = "df -h / | tail -n +2 | awk '{print $5}'" disk = ssh.run_command(command) logging.info("Disk usage is %s", disk) if int(disk.split('%')[0]) < int(diskt): command = "apt-get -y install " + install output = ssh.run_command(command) if output: PATH = os.getcwd() TEMPLATE_ENVIRONMENT = Environment( autoescape=False, loader=FileSystemLoader(os.path.join(PATH, 'templates')), trim_blocks=False) # Replace the following context from the template files context = {"DocumentRoot": documentroot, "ports": port} for files in os.listdir("templates"): logging.info("file name is %s", files) # Create new files from the template files after rendering create_file(files, context) indexdoc = documentroot + "/index.php" ssh1 = ssh.connection(name, user, passwd) scp = SCPClient(ssh1) # Copy the newly generated files scp.put('ports.conf', '/etc/apache2/ports.conf') scp.put('apache2.conf', '/etc/apache2/apache2.conf') scp.put('app.conf', '/etc/apache2/sites-available/app.conf') scp.put('resolv.conf', '/etc/resolv.conf') command = 'mkdir -p ' + documentroot output = ssh.run_command(command) scp.put('index.php', indexdoc) output = ssh.run_command( 'a2dissite 000-default && a2ensite app') scp.put('scripts/fix_iptables.sh', '/tmp/fix_iptables.sh') command = "/tmp/fix_iptables.sh " + port logging.info("running command %s", command) output = ssh.run_command(command) output = ssh.run_command("service apache2 stop") # We can implement override port here similar to disk threshold # (Not implemented correctly here) if override: command = "lsof -i :" + port + \ " | tail -n +2 | awk '{print $2}'| xargs kill -9" logging.info("running command %s", command) output = ssh.run_command(command) output = ssh.run_command('service apache2 start') hostconfig = hosts['cname'] + ":" + hosts['web_port'] vhost.append(hostconfig) ssh1.close() else: file.write( name + "::: installation failed please check output below \n") file.write(name + "::" + output + "\n") else: logging.exception( "Disk Usgae is above threshold.Please delete unwanted data and try agian" ) exit("Installation failed on host %s", name) print "vhost fils is ", vhost return vhost
def Download(self): # Start by setting a DL state self.inst['state'] = 9 # Downloading self.Save_Instance() # Do we have a temp folder locally we can store the files? savedir = download_dir + '/' + self.inst['uuid'] if not os.path.exists(savedir): os.makedirs(savedir, exist_ok=True) self.logger.debug("Download: Made save dir of " + savedir) # Are we in good, or error state? If error, DL error files our_dls = ast.literal_eval(self.inst['downloads']) if self.inst['backend_build_state'] == "failed": our_dls = ['/var/log/cloud-init-output.log', '/var/log/syslog'] # Before we start the downloads, open up an SSH session # SSH and check status ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() # Loop so we can monitor from this function, because f**k adding jank # to backend.py loop_counter = 0 while True: # Attempt our connection try: ssh.connect(self.inst['ip'], ssh_port, ssh_user, timeout=3) except Exception as e: # Did we except? Make sure it's a known message we can see msgs = [ "Unable to connect", "No existing session", "Authentication failed" ] for error in msgs: if error in str(e): ssh.close() # Cleanup time.sleep(1) # zzz to prevent racing continue # if we are here, we hit an unknown error. Have we been here # too many times? if loop_counter > 10: ssh.close() # Cleanup self.inst['state'] = 50 self.Save_Instance() raise Exception( "Error SSHing into droplet repeatedly, last error of " + str(e)) # Count up loop_counter += 1 continue else: # We connected, carry on to SCP for dl in ast.literal_eval(self.inst['downloads']): with SCPClient(ssh.get_transport(), socket_timeout=30) as scp: self.logger.debug( "Download: Downloading file/folder of " + dl) try: scp.get(dl, savedir, recursive=True) except Exception as e: self.inst['state'] = 50 self.Save_Instance() raise Exception( "Error with SCP Download, recieved an error of: " + str(e)) finally: # This should not be needed with the "with" # statement, but play it safe scp.close() finally: ssh.close() # If we are here, break out of the loop break # We done here return True
def scp_file_here(filename, name, user, password): print('{0} - SCP copy file here from router {1}'.format( _get_time(), filename)) ssh = createSSHClient(name, 22, user, password) with SCPClient(ssh.get_transport()) as scp: scp.get(filename)
def transfer_file(self, hostname=None, username=None, password=None, allow_agent=False, look_for_keys=False, ssh_config=None): """Transfer the file to the remote device over SCP. Note: If any arguments are omitted, the corresponding attributes of the ``self.device`` will be used. Args: hostname (str): OPTIONAL - The name or IP address of the remote device. proto (str): OPTIONAL - Protocol to be used for transfer - 'scp' or 'sftp' username (str): OPTIONAL - The SSH username for the remote device. password (str): OPTIONAL - The SSH password for the remote device. Raises: FileTransferError: if an error occurs during the file transfer. FileHashMismatchError: if the source and destination hashes don't match. FileNotReadableError: if the local file doesn't exist or isn't readable. FileNotEnoughSpaceError: if there isn't enough space on the device. FileRemoteDirDoesNotExist: if the remote directory doesn't exist. """ hostname = hostname or self.device.host username = username or self.device.username password = password or self.device.password ssh_config = ssh_config or self.device.ssh_config key_filename = None ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if ssh_config is not None: sshconfig = paramiko.SSHConfig() sshconfig.parse(open(ssh_config)) cfg = sshconfig.lookup('dev') key_filename = cfg['identityfile'][0] ssh.connect(hostname=hostname, username=username, password=password, port=self.port, allow_agent=allow_agent, look_for_keys=look_for_keys, key_filename=key_filename) if self.proto == "scp": scp = SCPClient(ssh.get_transport()) try: scp.get(self.src, self.dst) except: raise FileTransferError scp.close() elif self.proto == "sftp": sftp = ssh.open_sftp() sftp.sshclient = ssh try: sftp.get(self.src.strip('flash:'), self.dst) except: raise FileTransferError sftp.close() else: raise FileTransferUnsupportedProtocol(self.proto) src_hash = self._get_local_md5() dst_hash = self._get_remote_md5() if src_hash != dst_hash: raise FileHashMismatchError(self.src, self.dst, src_hash, dst_hash)
def _download_mlcnet_file(self, remote_file, download_to): scp = SCPClient(self.mlcnet_ssh.get_transport()) scp.get(remote_file, download_to) scp.close()
def deploy_orchestrator(self): """ Deploy Cloudify Manager. network, security group, fip, VM creation """ # network creation start_time = time.time() self.__logger.info("Creating keypair ...") kp_file = os.path.join(self.data_dir, "cloudify_ims.pem") keypair_settings = KeypairConfig(name='cloudify_ims_kp', private_filepath=kp_file) keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings) keypair_creator.create() self.created_object.append(keypair_creator) self.__logger.info("Creating full network ...") subnet_settings = SubnetConfig(name='cloudify_ims_subnet', cidr='10.67.79.0/24') network_settings = NetworkConfig(name='cloudify_ims_network', subnet_settings=[subnet_settings]) network_creator = OpenStackNetwork(self.snaps_creds, network_settings) network_creator.create() self.created_object.append(network_creator) ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds) router_creator = OpenStackRouter( self.snaps_creds, RouterConfig(name='cloudify_ims_router', external_gateway=ext_net_name, internal_subnets=[subnet_settings.name])) router_creator.create() self.created_object.append(router_creator) # security group creation self.__logger.info("Creating security group for cloudify manager vm") sg_rules = list() sg_rules.append( SecurityGroupRuleConfig(sec_grp_name="sg-cloudify-manager", direction=Direction.ingress, protocol=Protocol.tcp, port_range_min=1, port_range_max=65535)) sg_rules.append( SecurityGroupRuleConfig(sec_grp_name="sg-cloudify-manager", direction=Direction.ingress, protocol=Protocol.udp, port_range_min=1, port_range_max=65535)) securit_group_creator = OpenStackSecurityGroup( self.snaps_creds, SecurityGroupConfig(name="sg-cloudify-manager", rule_settings=sg_rules)) securit_group_creator.create() self.created_object.append(securit_group_creator) # orchestrator VM flavor self.__logger.info("Get or create flavor for cloudify manager vm ...") flavor_settings = FlavorConfig( name=self.orchestrator['requirements']['flavor']['name'], ram=self.orchestrator['requirements']['flavor']['ram_min'], disk=50, vcpus=2) flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings) flavor_creator.create() self.created_object.append(flavor_creator) image_settings = ImageConfig( name=self.orchestrator['requirements']['os_image'], image_user='******', exists=True) port_settings = PortConfig(name='cloudify_manager_port', network_name=network_settings.name) manager_settings = VmInstanceConfig( name='cloudify_manager', flavor=flavor_settings.name, port_settings=[port_settings], security_group_names=[securit_group_creator.sec_grp_settings.name], floating_ip_settings=[ FloatingIpConfig( name='cloudify_manager_fip', port_name=port_settings.name, router_name=router_creator.router_settings.name) ]) manager_creator = OpenStackVmInstance(self.snaps_creds, manager_settings, image_settings, keypair_settings) self.__logger.info("Creating cloudify manager VM") manager_creator.create() self.created_object.append(manager_creator) public_auth_url = keystone_utils.get_endpoint(self.snaps_creds, 'identity') self.__logger.info("Set creds for cloudify manager") cfy_creds = dict(keystone_username=self.snaps_creds.username, keystone_password=self.snaps_creds.password, keystone_tenant_name=self.snaps_creds.project_name, keystone_url=public_auth_url) cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip, username='******', password='******', tenant='default_tenant') self.orchestrator['object'] = cfy_client self.__logger.info("Attemps running status of the Manager") cfy_status = None retry = 10 while str(cfy_status) != 'running' and retry: try: cfy_status = cfy_client.manager.get_status()['status'] self.__logger.debug("The current manager status is %s", cfy_status) except Exception: # pylint: disable=broad-except self.__logger.warning("Cloudify Manager isn't " + "up and running. Retrying ...") retry = retry - 1 time.sleep(30) if str(cfy_status) == 'running': self.__logger.info("Cloudify Manager is up and running") else: raise Exception("Cloudify Manager isn't up and running") self.__logger.info("Put OpenStack creds in manager") secrets_list = cfy_client.secrets.list() for k, val in cfy_creds.iteritems(): if not any(d.get('key', None) == k for d in secrets_list): cfy_client.secrets.create(k, val) else: cfy_client.secrets.update(k, val) duration = time.time() - start_time self.__logger.info("Put private keypair in manager") if manager_creator.vm_ssh_active(block=True): ssh = manager_creator.ssh_client() scp = SCPClient(ssh.get_transport(), socket_timeout=15.0) scp.put(kp_file, '~/') cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/" run_blocking_ssh_command(ssh, cmd) cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem" run_blocking_ssh_command(ssh, cmd) cmd = "sudo yum install -y gcc python-devel" run_blocking_ssh_command( ssh, cmd, "Unable to install packages \ on manager") self.details['orchestrator'].update(status='PASS', duration=duration) self.vnf['inputs'].update( dict(external_network_name=ext_net_name, network_name=network_settings.name)) self.result = 1 / 3 * 100 return True
def upload(self, file, remote_folder): scp = SCPClient(self.ssh.get_transport()) scp.put(f'{file}', recursive=True, remote_path=remote_folder) scp.close()
'sudo pip install --upgrade google-api-python-client', 'sudo pip install beaker', 'sudo pip install peewee', 'sudo apt-get install unzip' ] for n, com in enumerate(commands): stdin, stdout, stderr = client.exec_command(com) if n == 1: # time.sleep(3) stdin.write('y\n') time.sleep(100) else: time.sleep(20) print "executed '" + com + "'" if __name__ == "__main__": scp = SCPClient(client.get_transport()) print "Uploading Webserver" scp.put(file_name) time.sleep(5) stdin, stdout, stderr = client.exec_command("unzip " + file_name) print "Setting up server" time.sleep(10) stdin, stdout, stderr = client.exec_command("cd " + file_name[:-4]) time.sleep(1) print "Executing" stdin, stdout, stderr = client.exec_command( "screen -d -m sudo python server.py") time.sleep(2) # print stdout.next() print "connection %s" % ("ok" if connection else "failed")
def deploy(path: str = './', hostname: str = "ev3dev", username: str = "robot", password: str = "maker", execute_file: Optional[str] = None, executable: List[str] = ('*.py',), exclude_path: str = "./.ignore", print_console: bool = True, redirect_stdout: bool = True, redirect_stderr: bool = True, redirect_stdin: bool = False) -> None: """ Send code to Ev3 :param path: The Directory to send (default is current directory). :param hostname: The ssh hostname (default is 'ev3dev') :param username: The ssh username (default is 'robot') :param password: The ssh password (default is 'maker') :param execute_file: A file to run on the ev3 when finished. 'None' to disable. Note: this file must be marked as executable. :param executable: A list of patterns of files that should be marked as executable (default is ['*.py']). :param exclude_path: The file containing the list of files to ignore (default is '.ignore'). :param print_console: Should we print info to the console? :param redirect_stdout: Should we redirect stdout form ev3 to console? :param redirect_stderr: Should we redirect stderr form ev3 to console? :param redirect_stdin: Should we redirect console input to ev3 stdin? This is disabled by default as it cannot terminate without reading from stdin. """ # Get / Set working directory if print_console: print("CD", path) os.chdir(path) working_dir = os.getcwd() dir_name = os.path.basename(working_dir) exclude = read_exclude(exclude_path) # Set up ssh if print_console: print("Starting ssh ...") ssh = SSHClient() ssh.load_system_host_keys() if print_console: print("Connecting to", F"{username}@{hostname} ...") ssh.connect(hostname=hostname, username=username, password=password) with SCPClient(ssh.get_transport()) as scp: for subdir, dirs, files in os.walk('.'): # for every file in current working directory: for filename in files: filepath = subdir + '/' + filename # get full file path (relative to working directory) if not match(filepath, exclude): # if the file path does not match any of the excluded patterns: if print_console: print("Sending", Path(filepath), "... ", end='') # create the directory if it does not exist ssh.exec_command('mkdir -p ' + path_join('~', dir_name, subdir).as_posix()) # copy files using scp scp.put(str(path_join(working_dir, filepath)), path_join('~', dir_name, filepath).as_posix()) if print_console: print("Sent") if match(filepath, executable): # if file path matches any of the executable patterns: # mark as executable if print_console: print(path_join('~', dir_name, filepath).as_posix(), "marked as executable.") ssh.exec_command('chmod u+x ' + path_join('~', dir_name, filepath).as_posix()) else: if print_console: print('Excluding', Path(filepath), '.') if execute_file: if print_console: print(F'\nExecuting {execute_file} ...\n') # execute the file. stdin, stdout, stderr = ssh.exec_command(path_join('~', dir_name, execute_file).as_posix(), get_pty=True) # create the redirecting threads if redirect_stdout: out = threading.Thread(target=redirect_stdout_handler, args=(stdout,)) if redirect_stderr: err = threading.Thread(target=redirect_stderr_handler, args=(stderr,)) if redirect_stdin: child_pid = os.fork() if child_pid == 0: redirect_stdin_handler(stdin) os.kill(os.getpid(), SIGTERM) # sin = threading.Thread(target=redirect_stdin_handler, args=(stdin,)) # start them if redirect_stdout: out.start() if redirect_stderr: err.start() # if redirect_stdin: # sin.start() # wait for them to terminate if redirect_stdout: out.join() if redirect_stderr: err.join() if redirect_stdin: global run_stdin # tell redirect_stdin_handler to exit without sending data to stdin run_stdin = False # sys.stdin.close() # sin.join() os.kill(child_pid, SIGTERM) if print_console: print('\nFinished.')
def main(username, password, test=False, force=False): ssh = SSHClient() ssh.set_missing_host_key_policy(AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect('nova.cs.tau.ac.il', username=username, password=password) scp = SCPClient(ssh.get_transport(), progress = progress) # copy benchmark unbound receptor and ligand to nova: all_complex_codes = next(os.walk('../data'))[1] if test: all_complex_codes = all_complex_codes[:1] with open('./copied_successfully.json', 'r') as f: copied_successfully = json.load(f) if force: copied_successfully = [] for complex_code in all_complex_codes: if len(complex_code) != 4: continue print("Copying complex: %s" % complex_code) try: ssh.exec_command('mkdir ~/data/' + complex_code + '/') ssh.exec_command('mkdir ~/data/' + complex_code + '/benchmark/') ssh.exec_command('mkdir ~/data/' + complex_code + '/patch_dock/') scp.put('../data/' + complex_code + '/benchmark/' + complex_code + '_l_u.pdb', '~/data/' + complex_code + '/benchmark/') scp.put('../data/' + complex_code + '/benchmark/' + complex_code + '_r_u.pdb', '~/data/' + complex_code + '/benchmark/') scp.put("../data/{0}/patch_dock/{0}.patch_dock_output".format(complex_code), '~/data/' + complex_code + '/patch_dock/') copied_successfully.append(complex_code) with open('./copied_successfully.json', 'w') as f: json.dump(copied_successfully, f) except: print("ERROR with: %s" % complex_code) # run patch dock on each benchmark receptor and ligand which doesn't have an ouput file yet with open('./patch_dock_ran_successfully.json', 'r') as f: patch_dock_ran_successfully = json.load(f) if force: patch_dock_ran_successfully = [] for complex_code in all_complex_codes: if len(complex_code) != 4 or complex_code in patch_dock_ran_successfully: continue print("Running PatchDock on: %s" % complex_code) try: ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('cd ~/PatchDock; sh run_patch_dock_on_one_complex.sh ' + complex_code) print(ssh_stdout.read(), ssh_stderr.read()) if ssh_stderr.read() == b'': patch_dock_ran_successfully.append(complex_code) with open('./patch_dock_ran_successfully.json', 'w') as f: json.dump(patch_dock_ran_successfully, f) except Exception as e: print(e.message) print("ERROR with: %s" % complex_code) print("DONE") scp.close()
def put(self, files, remote_path=b'.', recursive=False): client = self._get_client() with SCPClient(client.get_transport()) as scp: scp.put(files, remote_path, recursive)
def connect(self): """ Establish a connection to the device. Purpose: This method is used to make a connection to the junos | device. The internal property conn_type is what | determines the type of connection we make to the device. | - 'paramiko' is used for operational commands (to allow | pipes in commands) | - 'scp' is used for copying files | - 'shell' is used for to send shell commands | - 'root' is used when logging into the device as root, and | wanting to send operational commands | - 'ncclient' is used for the rest (commit, compare_config, | commit_check) @returns: None @rtype: None """ if self.conn_type == 'paramiko': self._session = paramiko.SSHClient() # These two lines set the paramiko logging to Critical to # remove extra messages from being sent to the user output. logger = logging.Logger.manager.getLogger('paramiko.transport') logger.setLevel(logging.CRITICAL) self._session.set_missing_host_key_policy( paramiko.AutoAddPolicy()) self._session.connect(hostname=self.host, username=self.username, password=self.password, port=self.port, timeout=self.connect_timeout) if self.conn_type == 'scp': self._scp_session = paramiko.SSHClient() logger = logging.Logger.manager.getLogger('paramiko.transport') logger.setLevel(logging.CRITICAL) self._scp_session.set_missing_host_key_policy( paramiko.AutoAddPolicy()) self._scp_session.connect(hostname=self.host, username=self.username, password=self.password, port=self.port, timeout=self.connect_timeout) self._scp = SCPClient(self._scp_session.get_transport()) elif self.conn_type == "ncclient": self._session = manager.connect( host=self.host, port=self.port, username=self.username, password=self.password, timeout=self.connect_timeout, device_params={'name': 'junos'}, hostkey_verify=False ) elif self.conn_type == 'shell': if not self._session: self.conn_type = 'paramiko' self.connect() self.conn_type = 'shell' if not self._shell: self._shell = self._session.invoke_shell() time.sleep(2) if self.username != 'root' and not self._in_cli: self._in_cli = True if not self.cli_to_shell(): self._shell.recv(9999) elif self.conn_type == 'root': # open the shell if necessary, and move into CLI if not self._shell: self._shell = self._session.invoke_shell() time.sleep(2) if not self.shell_to_cli(): self._shell.recv(9999) self._update_timeout(self.session_timeout)
def get(self, remote_path, local_path='/tmp/', recursive=True): client = self._get_client() with SCPClient(client.get_transport()) as scp: scp.get(remote_path, local_path, recursive)
cur_time = int(time.time()) while cur_time < timeout: if IsOnline(sync_ip): break cur_time = int(time.time()) if not IsOnline(sync_ip): print("Failed to wake %s" % sync_ip) time.sleep(60) continue ssh = SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(sync_ip, username=username) scp = SCPClient(ssh.get_transport(), progress=progress) sync(monitor_dir, sync_dir, scp) scp.close() ssh.close() # if there are, pause main queue and sync them to NAS # resume when sync complete time.sleep(60)
def cpfiles(self): with SCPClient(ssh.get_transport()) as scp: scp.put('a.csv', 'b.csv') scp.get('23.txt')
with tarfile.open( f"/tmp/packetframedeploy/{node['name']}.tar.gz", "w:gz") as tar: output_file = f"/tmp/packetframedeploy/{node['name']}/" tar.add(output_file, arcname=os.path.basename(output_file)) try: ssh.connect(node["management_ip"], username="******", port=34553, key_filename=configuration["ssh-key"]) except (TimeoutError, NoValidConnectionsError): print(node["name"] + " unable to connect.") else: with SCPClient(ssh.get_transport()) as scp: scp.put( f"/tmp/packetframedeploy/{node['name']}.tar.gz", "/etc/bind/deploy.tar.gz") stdin, stdout, stderr = ssh.exec_command( "tar -xvzf /etc/bind/deploy.tar.gz -C /etc/bind/ ; rm /etc/bind/deploy.tar.gz ; bash /root/cleanup-zones.sh ; rndc reload" ) for line in stdout: print(line.strip()) if line in stderr: print(line.strip()) # Clean up the tmp files shutil.rmtree("/tmp/packetframedeploy/")
def _publish_code(package_type, d_files, package_name, ssh, ip, ssh_port, app_port, package_path, package_md5,execute): #上线操作前进行数据备份 _console_out(channel, Msg_Key,"ip:%s ssh_port:%s app_port:%s --->start deploy %s......" % (ip, ssh_port, app_port,project)) if execute == 'publish': try: cmd = "[ -e %s%s ] && echo ok" % (web_path, package_name) stdin, stdout, stderr = ssh.exec_command(cmd) result = str(stdout.read().strip(),encoding='utf8') if result == 'ok': if package_name.endswith('.war'): cmd = "\cp -rf %s%s %s%s" % (web_path, package_name, bak_path,package_name) else: cmd = "/usr/bin/rsync -av --delete %s%s/ %s%s/" % (web_path, package_name, bak_path, package_name) stdin, stdout, stderr = ssh.exec_command(cmd) result = stderr.read() if result: _flow_log(flow_number,"Error:%s"%result) return "ip:%s ssh_port:%s app_port:%s --->backup Fail !" % (ip, ssh_port, app_port) except Exception as e: _flow_log(flow_number,'Error:%s'%str(e)) return "ip:%s ssh_port:%s app_port:%s --->backup Fail !" % (ip, ssh_port, app_port) scp = SCPClient(ssh.get_transport()) #增量包部署 if package_type == 'part': try: scp.put("%s/"%d_files, "%s%s/" % (web_path, package_name), recursive=True) except Exception as e: # 传输错误重试3次 for i in range(3): time.sleep(3) try: scp.put("%s/" % d_files, "%s%s/" % (web_path, package_name), recursive=True) except: if i >=2: break continue else: cmd = "chown %s:%s -R %s%s/" % (service_user, service_user, web_path, package_name) stdin, stdout, stderr = ssh.exec_command(cmd) result = stderr.read() if result: _flow_log(flow_number, "Error:%s" % result) return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port, web_path, package_name) _flow_log(flow_number,'Error:%s'%str(e)) return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port) else: cmd = "chown %s:%s -R %s%s/" %(service_user,service_user,web_path, package_name) stdin, stdout, stderr = ssh.exec_command(cmd) result = stderr.read() if result: _flow_log(flow_number, "Error:%s" % result) return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port,web_path, package_name) #整包部署 if package_type == 'full': try: d_zip = "%s%s" % (web_path, package_path.split('/')[-1]) try: scp.put(package_path, d_zip) except Exception as e: # 传输错误重试3次 for i in range(3): time.sleep(3) try: scp.put(package_path, d_zip) except: if i >= 2: break continue _flow_log(flow_number,'Error:%s'%str(e)) return "ip:%s ssh_port:%s app_port:%s --->transfers Fail !" % (ip, ssh_port, app_port) cmd = '/usr/bin/md5sum %s' % d_zip stdin, stdout, stderr = ssh.exec_command(cmd) R_md5 = str(stdout.read().split()[0],encoding='utf8') if R_md5 == package_md5: package_zip = package_path.split('/')[-1] cmd = "cd %s && /usr/bin/unzip -qo %s && /bin/rm -f %s && [ -e %s ] && echo ok" % (web_path,package_zip,package_zip,package_zip.replace('.zip', '')) stdin, stdout, stderr = ssh.exec_command(cmd) result_zip = str(stdout.read().strip(),encoding='utf8') result = stderr.read() if result_zip == 'ok': cmd = "cd %s && /bin/rm -rf %s{,.war} &&/bin/mv %s %s" % (web_path,project,package_zip.replace('.zip', ''), package_name) stdin, stdout, stderr = ssh.exec_command(cmd) result = stderr.read() if result: _flow_log(flow_number,"Error:%s"%result) return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port) else: cmd = "chown %s:%s %s%s" % (service_user, service_user, web_path, package_name) stdin, stdout, stderr = ssh.exec_command(cmd) result = stderr.read() if result: _flow_log(flow_number, "Error:%s" % result) return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port, web_path, package_name) else: _flow_log(flow_number, "Error:%s" % result) return "ip:%s ssh_port:%s app_port:%s --->unzip Fail !" % (ip, ssh_port, app_port) else: return "ip:%s ssh_port:%s app_port:%s --->md5 Fail !" % (ip, ssh_port, app_port) except Exception as e: if 'old-style' not in str(e): _flow_log(flow_number,'Error:%s'%str(e)) return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
from paramiko import SSHClient from scp import SCPClient import tarfile, os, shutil, time overall_time = time.time() start_time = time.time() ssh = SSHClient() ssh.load_system_host_keys() ssh.connect('34.72.152.18', username="******") print("ssh connection took: ", time.time() - start_time) start_time = time.time() # SCPCLient takes a paramiko transport as an argument scp = SCPClient(ssh.get_transport()) scp.get("/home/marekjachym99/lavalanche/weather.tar", "../data") scp.close() print("scp copying took: ", time.time() - start_time) # extract .tar file start_time = time.time() archive = tarfile.open("../data/weather.tar", "r") archive.extractall("../data/") archive.close() if os.path.isdir("../data/weather_data"): shutil.rmtree("../data/weather_data") shutil.move("../data/home/marekjachym99/lavalanche/weather_data", "../data/") os.remove("../data/weather.tar") shutil.rmtree("../data/home") print("file handling took: ", time.time() - start_time) print("whole process took: ", time.time() - overall_time)
def ltm(IP, ACC, PASS): # 找出錯誤log並儲存到 \IP_ERR_LOG中 try: client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(IP, username=ACC, password=PASS) # 以paramiko進行ssh連線 _, stdout, _ = client.exec_command( "cd /var/log; ls") # 列出 /var/log目錄下所有檔案 log = "" lst = [ line.replace('\n', '') for line in stdout.readlines() if line[:3] == "ltm" ] # 尋找ltm log並儲存到lst中 scp = SCPClient(client.get_transport()) # 準備以scp協定傳輸檔案 for line in lst: scp.get("/var/log/" + line, PATH + "\\" + IP + "_log\\" + line) # 取得/var/log中的ltm log if line[-2:] == "gz": # 如果該log為壓縮擋 則解壓縮再讀取 with gzip.open(IP + "_log\\" + line, "rb") as f_in: log += f_in.read().decode() # log字串增加讀取到的檔案內容 else: # 如果該log不是壓縮擋 則直接讀取 with open(IP + "_log\\" + line, "rb") as f_in: log += f_in.read().decode() # log字串增加讀取到的檔案內容 except Exception as e: print(e) return # == HA state change P = re.compile("\n.*HA unit.*\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef: ef.writelines(res) P = re.compile("\n.*No failover status messages received for.*\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef: ef.writelines(res) P = re.compile("\n.*Active\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef: ef.writelines(res) P = re.compile("\n.*Offline\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef: ef.writelines(res) P = re.compile("\n.*Standby\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\HA_ERR.log", "a", newline='') as ef: ef.writelines(res) # == VS state change P = re.compile("\n.*Virtual Address .*GREEN to RED.*\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\VS_ERR.log", "w", newline='') as ef: ef.writelines(res) # == Pool P = re.compile("\n.*Pool.*GREEN to RED.*\n") res = re.findall(P, log) if len(res) != 0: if not os.path.exists(IP + "_ERR_LOG"): os.makedirs(IP + "_ERR_LOG") with open(IP + "_ERR_LOG\\Pool_ERR.log", "w", newline='') as ef: ef.writelines(res)