def execute(self, client, stage_number): """ Move the tar file into place, unpack and run bootstrap.sh passing any given arguments. """ sftp = SFTPClient.from_transport(client.get_transport()) stage_directory = 'stage-' + str(stage_number) try: sftp.mkdir(stage_directory) except: logger.exception('') sftp.chdir(stage_directory) # Stage complete so nothing to do. if 'stage-complete' in sftp.listdir(): return # upload, untar, execute bootstrap.sh sftp.put(self.tarfile, '{0}/{1}.tar'.format(sftp.getcwd(), stage_directory)) untar = 'cd {0} && tar xf {1}.tar'.format(stage_directory, stage_directory) untar_result = self.execute_command(untar, client) logger.debug('Command results: {0}.'.format(untar_result)) logger.info('Executing contents of tar file: {0}.'.format(self.tarfile)) bootstrap_arguments = ' '.join(self.args) run_bootstrap = 'cd {0} '.format( stage_directory) + '&& sudo -u root -H bash -l -c "bash bootstrap.sh {0}" > output '.format( bootstrap_arguments) + '&& touch stage-complete' bootstrap_result = self.execute_command(run_bootstrap, client) if bootstrap_result[1] != 0: logger.error("Stage did not complete: {0}.".format(stage_number)) logger.debug('Command results: {0}.'.format(bootstrap_result[0]))
def apply_ericomshield_version(): with tarfile.open('./system.tar.gz'.format(ericom_shield_install_dir), mode='w:gz') as tar: for file in glob.glob("{}/*".format(ericom_shield_install_dir)): tar.add(file) for file in glob.glob('{}/.*'.format(ericom_shield_install_dir)): if not '.es_ip_address' in file: tar.add(file) transport = client.get_transport() sftp_client = SFTPClient.from_transport(transport) sftp_client.put( './system.tar.gz'.format(ericom_shield_install_dir), '/home/{}/system.tar.gz'.format(os.environ['MACHINE_USER'])) sftp_client.close() _, stdout, stderr = client.exec_command( "tar xfz ./system.tar.gz && sudo rsync -avh ./usr/local/ericomshield/ /usr/local/ericomshield/ && rm -rf ./usr ./system.tar.gz" ) while not stdout.channel.exit_status_ready(): one_line = '' if stdout.channel.recv_ready(): one_line = stdout.channel.recv(2048).decode("ascii") sys.stdout.write(one_line) sys.stdout.flush() output = subprocess.check_output( 'rm -f ./system.tar.gz'.format(ericom_shield_install_dir), shell=True)
def remote_scp(ftp_type, host_ip, remote_path, local_path, username, password): ssh_port = 22 try: conn = Transport((host_ip, ssh_port)) conn.connect(username=username, password=password) sftp = SFTPClient.from_transport(conn) if ftp_type == 'remoteRead': print('read') if not local_path: filename = os.path.split(remote_path) local_path = os.path.join('/tmp', filename[-1]) print('开始从服务器下载文件......') sftp.get(remote_path, local_path) print(f'文件{filename[-1]}已经下载到本地') if ftp_type == "remoteWrite": print('write') sftp.put(local_path, remote_path) conn.close() return True except IOError as e: print('没有找到目录', e) except Exception as e: print('error!!!', e)
def restore(path): dbinfo = choicedb(path+'/conf/db_info.yaml') editData(dbinfo, [['reset master;']]) if sys.platform == 'win32': t = Transport(dbinfo[0], 22) t.connect(username=dbinfo[5], password=dbinfo[6]) sftp = SFTPClient.from_transport(t) src = '/home/mysql/basedata/initdata.sql' des = path + '/database/initdata.sql' sftp.put(des,src ) t.close() else: child = spawn( 'scp '+path+'/database/initdata.sql '+ dbinfo[5]+'@'+dbinfo[0]+':/home/mysql/basedata/') child.expect("password:"******"'"+dbinfo[3]+"'"' '+dbinfo[4]+' </home/mysql/basedata/initdata.sql') return stderr.read().decode('gbk', errors='ignore')
def dbbackup(path): dbinfo = choicedb(path + '/conf/db_info.yaml') s = sshconnect(hostname=dbinfo[0], port=dbinfo[7], username=dbinfo[5], password=dbinfo[6]) st, std, stde = s.exec_command('cd /home/mysql/backup'+'\n' 'ls') filelist = std.read().decode('gbk',errors='ignore').split('\n') if 'databackup.sql' in filelist: s.exec_command('cd /home/mysql/backup'+'\n' 'rm -rf databackup1.sql') stdin, stdout, stderr = s.exec_command('mysqldump -t -u'+dbinfo[2]+' -p'+dbinfo[3]+' '+dbinfo[4]+' >/home/mysql/backup/databackup.sql') if 'error' not in stderr.read().decode('gbk', errors='ignore'): t1,t2,t3=s.exec_command('du -b /home/mysql/databackup.sql') if sys.platform == 'win32': t = Transport(dbinfo[0], 22) t.connect(username=dbinfo[5], password=dbinfo[6]) sftp = SFTPClient.from_transport(t) src = '/home/mysql/backup/databackup.sql' des = path + '/database/databackup.sql' sftp.get(src, des) t.close() else: child = spawn( 'scp ' + dbinfo[5] + '@' + dbinfo[0] + ':/home/mysql/backup/databackup.sql ' + path + '/database/') child.expect("password:") child.sendline(dbinfo[6]) child.read() with zipfile.ZipFile(path + '/database/backupdatabase.zip', 'w') as z: z.write(path + '/database/databackup.sql') return t2.read().decode('gbk',errors='ignore').split('\t')[0].strip() else: return False s.close
def _authenticate(self): self._transport = SFTPTransport((self.config['sftp_host'], self.config['sftp_port'])) self._transport.connect(username=self.config['sftp_user'], password=self.config['sftp_password']) self.session = SFTPClient.from_transport(self._transport) logging.info('SFTP Authorization succeed')
def prepare_machine_to_docker_node(ip): _, stdout, _ = client.exec_command('hostname') hostname = (stdout.read()).decode('ascii').replace('\n', '') with open('hosts', mode='w') as file: file.write("{0}\t{1}\n".format(ip, hostname)) for manager in get_managers_ip_and_name(): file.write('{}\n'.format(manager)) file.close() transport = client.get_transport() sftp_client = SFTPClient.from_transport(transport) # sftp_client.put(os.path.abspath('./hosts'),'/home/{}/hosts'.format(os.environ['MACHINE_USER'])) # stdin, stdout, stderr = client.exec_command('cat ./hosts | sudo tee -a /etc/hosts && rm -f ./hosts') # stdout.channel.recv_exit_status() # logger.error(stderr.read()) sftp_client.put( os.path.abspath('./sysctl_shield.conf'), '/home/{}/sysctl_shield.conf'.format(os.environ['MACHINE_USER'])) sftp_client.put( os.path.abspath('./mount-tmpfs-volume.sh'), '/home/{}/mount-tmpfs-volume.sh'.format(os.environ['MACHINE_USER'])) sftp_client.close() stdin, stdout, stderr = client.exec_command( 'chmod +x ./mount-tmpfs-volume.sh && sudo ./mount-tmpfs-volume.sh && rm -f ./mount-tmpfs-volume.sh' ) stdout.channel.recv_exit_status() logger.error(stderr.read()) stdin, stdout, stderr = client.exec_command( 'cat ./sysctl_shield.conf | sudo tee "/etc/sysctl.d/30-ericom-shield.conf"; sudo sysctl --load="/etc/sysctl.d/30-ericom-shield.conf"' ) stdout.channel.recv_exit_status() logger.error(stderr.read()) return hostname
def install_docker(): transport = client.get_transport() sftp_client = SFTPClient.from_transport(transport) sftp_client.put( os.path.abspath('./install-docker.sh'), '/home/{}/install-docker.sh'.format(os.environ['MACHINE_USER'])) sftp_client.close() running = True channel = client.get_transport().open_session() channel.exec_command( 'chmod +x install-docker.sh && sudo ./install-docker.sh && rm -f ./install-docker.sh' ) print('Install docker please wait...', end='') while running: sys.stdout.write('.') sys.stdout.flush() time.sleep(1) if channel.exit_status_ready(): print(' ') if channel.recv_exit_status() == 0: print("Docker installed") else: print("Docker installation failed") running = False if test_docker_on_machine(): logger.info("Docker installation success") else: logger.error("Docker installation failed") sys.exit(1)
def check_destination_path(self, path, mkdir_destination=None, ssh=None): """Returns the destination_dir or raises an Exception if destination_dir does not exist on the remote host. @param path: path on remote host @type path: byte or str @param mkdir_destination: if True attempts to create the remote folder. @type mkdir_destination: boolean @raise FileNotFoundError: if path does not exist and mkdir_destination is False @return path """ ssh = ssh if ssh else self.ssh if path[0:1] == b'~' or path[0:1] == '~': _, stdout, _ = ssh.exec_command("pwd") path = os.path.join(stdout.readlines()[0].strip(), path.replace('~/', '')) with SFTPClient.from_transport(ssh.get_transport()) as sftp: try: sftp.chdir(path) except IOError: if mkdir_destination or self.mkdir_destination: self.mkdir_p(sftp, path) else: raise FileNotFoundError( '{} not found on remote host.'.format(path)) return path return None
def connect(self): host = self.server.host.split(':') port = 21 if self.mode == 0 else 22 if len(host) > 1: try: port = int(host[1]) except: message = log.Message('port: %s is invalid.' % host[1], log.Level.ERROR) message.log() return False if self.mode == 0: self.session = ftplib.FTP() self.session.connect(host[0], port) self.session.login(self.server.user, self.server.password) else: ssh = SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(host[0], port, username=self.server.user, password=self.server.password) self.session = SFTPClient.from_transport(ssh.get_transport()) return True
def connect(self): """ Get a handle to a remote connection """ # Check URL schema = urlparse(self.url) if schema.scheme == 'sftp': self.transport = Transport((schema.hostname, int(schema.port))) else: raise SFTPError('Not a valid sftp url %s, type is %s' % (self.url, schema.scheme)) # Add authentication to transport try: if self.password: self.transport.connect(username=self.user_name, password=self.password) elif self.private_key: self.transport.connect(username=self.user_name, pkey=RSAKey.from_private_key( StringIO(self.private_key))) else: raise SFTPError("No password or private_key defined") # Connect self.conn = SFTPClient.from_transport(self.transport) except (socket.gaierror, error), msg: raise SFTPError( str(msg) + ' while establishing connection to %s' % (self.url, ))
def _connect(self): """Open connection to remote host.""" if self.conn is None: try: self.client = SSHClient() self.client.load_system_host_keys() self.client.set_missing_host_key_policy(AutoAddPolicy()) self.client.connect(self.host, username=self.user, key_filename=self.ssh_key_filepath, look_for_keys=True, banner_timeout=2000, auth_timeout=2000, timeout=5000) # Define progress callback that prints the current percentage completed for the file def progress(filename, size, sent): print("%s's progress: %.2f%% \r" % (filename, float(sent) / float(size) * 100), end='\x1b[1K\r') progress_val = progress if self.show_progress else None self.scp = SCPClient(self.client.get_transport(), progress=progress_val, socket_timeout=15.0) self.sftp = SFTPClient.from_transport( self.client.get_transport()) except AuthenticationException as error: logger.error(f'Authentication failed: \ did you remember to create an SSH key? {error}') raise error return self.client
def __init__(self, host, port=22, user='******', progress=None): self.ssh = SSHClient() self.ssh.load_system_host_keys() self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.ssh.connect(host, port, user) self.sftp = SFTPClient.from_transport(self.ssh.get_transport()) self.scp = SCPClient(self.ssh.get_transport(), progress=progress)
def connect_remote(self): # open connection to remote host try: self.client = SSHClient() self.client.load_system_host_keys() self.client.set_missing_host_key_policy(AutoAddPolicy()) self.client.connect(self.host, port=22, username=self.user, password=self.password, look_for_keys=False, timeout=10) self.scp = SCPClient(self.client.get_transport()) sftp = SFTPClient.from_transport(self.client.get_transport()) try: sftp.stat(self.remote_path) except FileNotFoundError: raise AuthenticationException("Remote path does not exist") try: sftp.stat(self.index_path) except FileNotFoundError: raise AuthenticationException("Index path does not exist") return self.client except AuthenticationException as error: logger.info( 'Authentication failed: did you enter the correct username and password?' ) logger.error(error) self.scp = None return error
def write_redirects_to_sftp(self, from_path, to_path, cron): try: ssh_key_object = RSAKey(filename=app.config['SFTP_SSH_KEY_PATH'], password=app.config['SFTP_SSH_KEY_PASSPHRASE']) remote_server_public_key = HostKeyEntry.from_line(app.config['SFTP_REMOTE_HOST_PUBLIC_KEY']).key # This will throw a warning, but the (string, int) tuple will automatically be parsed into a Socket object remote_server = Transport((app.config['SFTP_REMOTE_HOST'], 22)) remote_server.connect(hostkey=remote_server_public_key, username=app.config['SFTP_USERNAME'], pkey=ssh_key_object) sftp = SFTPClient.from_transport(remote_server) sftp.put(from_path, to_path) if cron: return 'SFTP publish from %s to %s succeeded' % (from_path, to_path) else: return fjson.dumps({ 'type': 'success', 'message': 'Redirect updates successful' }) except: if cron: return 'SFTP publish from %s to %s failed' % (from_path, to_path) else: return fjson.dumps({ 'type': 'danger', 'message': 'Redirect updates failed' })
def run(cls, connection, wspaceFolder, stmt): if connection.transport is None: raise ConnectionClosedError lootFolder = join(wspaceFolder, "loot", str(connection.endpoint).replace(':', '-'), "") if not exists(lootFolder): mkdir(lootFolder) filepath = getattr(stmt, 'file', None) if filepath is None: print("You must specify a path") return False filedest = join(lootFolder, filepath.replace('/', '_')) #TODO err management sftp = SFTPClient.from_transport(connection.transport) print("Retreiving file " + filepath + "... ", end="") sys.stdout.flush() try: sftp.get(filepath, filedest) except Exception as e: print("Error " + str(type(e)) + ": " + str(e)) return False print("Done") print("File saved as " + filedest)
def _get_ftp_client(self, window_size=None, max_packet_size=None, buffer_size=None): """ opens an sftp or scp channel and closes it after execution finished Returns: """ window_size = window_size or settings.SFTP_SCP_WINDOW_SIZE max_packet_size = max_packet_size or settings.SFTP_SCP_MAX_PACKET_SIZE buffer_size = buffer_size or settings.SFTP_SCP_BUFFER_SIZE c = None try: if self.file_transfer_protocol == 'sftp': c = SFTPClient.from_transport(self.ssh_transport, window_size=window_size, max_packet_size=max_packet_size) else: c = scp.SCPClient(self.ssh_transport, buff_size=buffer_size) yield c finally: try: if c: c.close() except Exception: log.exception('problems closing sftp/scp client')
def _copy_to_server(self, filename, destination=None): self._ensure_connected() if destination is None: destination = self.working_directory sftp = SFTPClient.from_transport(self.ssh.get_transport()) # tar and compress a directory before copying if os.path.isfile(filename): sftp.put(filename, os.path.join(destination, os.path.basename(filename))) elif os.path.isdir(filename): tar_filename = filename + '_' + str(uuid.uuid4()) + '.tar.gz' self.logger.info("Compressing " + filename + '...') subprocess.call(['tar', 'cfz', tar_filename, filename]) self.logger.info("Copying " + tar_filename + " to " + self.hostname + ':' + os.path.join(destination, os.path.basename(tar_filename)) + '...') sftp.put(tar_filename, os.path.join(destination, os.path.basename(tar_filename))) os.remove(tar_filename) _, stdout, stderr = self._run_command('cd ' + quote(self.working_directory) + ' && tar xvfz ' + quote(tar_filename) + ' && rm ' + quote(tar_filename)) self.logger.debug("tar STDOUT: " + stdout.read()) self.logger.debug("tar STDERR: " + stderr.read()) else: raise NotImplementedError("Can't SFTP put anything other than a folder or file yet.") sftp.close()
def upload(dest, srcPath, destPath, callBack, skipIfExists = False): try: sftp = SFTPClient.from_transport(dest.connection) # This is pretty awful, if the file doesn't exists it throws # an exception, and then we should continue # Assume true exists = True if skipIfExists: try: sftp.stat(destPath) except FileNotFoundError as e: exists = False if not skipIfExists or not exists: renameUpload(sftp, srcPath, destPath, callBack) else: callBack() return True except Exception as e: print('*** Caught exception: %s: %s' % (e.__class__, e)) traceback.print_exc() sys.exit(1) finally: try: t.close() except: pass return False
def __init__(self, host, port, username, password): self.host = host self.port = port self.username = username self.password = password self.t = Transport((self.host, self.port)) self.t.connect(username=self.username, password=self.password) # 注意:此处需要用默认传参方式传参 self.sftp = SFTPClient.from_transport(self.t)
def scp(self, srcFile, destPath): transport = Transport((self.host, int(self.port))) transport.connect(username=self.user, password=self.passwd) sftp = SFTPClient.from_transport(transport) try: sftp.put(srcFile, destPath) except IOError as e: raise e
def scp(self, src_file, dest_path): transport = Transport((self.host, int(self.port))) transport.connect(username=self.user, password=self.passwd) sftp = SFTPClient.from_transport(transport) try: sftp.put(src_file, dest_path) except IOError as e: raise e
def __init__(self, user, passwd, ip): self.ip = ip try: t = Transport((ip, 22)) t.connect(username=user, password=passwd) self.sftpObject = SFTPClient.from_transport(t) self.status = 'Success' except: self.status = 'Failed'
def __init__(self, host, username, password): try: logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.INFO) transport = Transport(sock=(host)) transport.connect(username=username, password=password) self.connection = SFTPClient.from_transport(transport) except: logging.error('Can not able to conenct to server')
def _get_sftp_connection(host): transport = host.connection.get_transport() try: return SFTPClient.from_transport(transport) except SSHException as e: raise ConnectError(( "Unable to establish SFTP connection. Check that the SFTP subsystem " "for the SSH service at {0} is enabled.").format(host), ) from e
def sftpclient(sftpserver): from paramiko import Transport transport = Transport((sftpserver.host, sftpserver.port)) transport.connect(username="******", password="******") sftpclient = SFTPClient.from_transport(transport) yield sftpclient sftpclient.close() transport.close()
def copy(self, local_file, remote_file): if self.localMachine: command = "cp %s %s" % (local_file, remote_file) return self.execute_command(command) else: if not hasattr(self, 'ssh_connection'): self.__get_ssh_connection() sftp = SFTPClient.from_transport(self.ssh_connection.get_transport()) obj = sftp.put(local_file, remote_file) return (0, "Copied to %s" % obj.__str__(), "")
def __init__(self, sftp_settings): self.transport = Transport( (sftp_settings['HOST'], int(sftp_settings['PORT']))) self.transport.connect(username=sftp_settings['USER'], password=sftp_settings['PASSWORD']) self.connection = SFTPClient.from_transport(self.transport) logger.debug( "SFTPUploader initiated. Sending files to {host}:{port}".format( host=sftp_settings['HOST'], port=sftp_settings['PORT']))
def _get_sftp_connection(host): transport = host.connection.get_transport() try: return SFTPClient.from_transport(transport) except SSHException as e: six.raise_from( ConnectError(( 'Unable to establish SFTP connection. Check that the SFTP subsystem ' 'for the SSH service at {0} is enabled.').format(host)), e)
def sftp_authentication(self): try: transport = Transport((ip_server, 222)) privatekeyfile = os.path.expanduser('./priv_key') mykey = RSAKey.from_private_key_file(privatekeyfile) transport.connect(username='******', pkey=mykey) sftp_client = SFTPClient.from_transport(transport) return sftp_client, transport except Exception, e: print "SFTP Authentication Fail:" + str(e)
def transport(self): transport = Transport((self.host, self.port)) transport.connect(username=self.username, password=self.password) sftp = SFTPClient.from_transport(transport) sftp.put(f'{self.project_url}/web/caweb/html.zip', f'{self.nginx_url}/html.zip') sftp.put(f'{self.project_url}/server/caserver.zip', f'{self.nginx_url}/server/new_caserver.zip') print("transport finished!") transport.close()
def __init__(self, config): try: transport = Transport((config.SFTP_HOST, config.SFTP_PORT)) transport.connect(username=config.SFTP_USERNAME, password=config.SFTP_PASSWORD) self.sftp_client = SFTPClient.from_transport(transport) except (SSHException, socket.error) as e: raise SFTPError(f'SFTP connection failed due to: {e}.') self.path = config.SFTP_PATH self.temp_file_path = config.TEMP_FILE_PATH
def scp_remote_to_local(remote_file_path, local_file_path, ip, username, password): """Copies a file from remote to local""" if not username: username = DEFAULT_CVM_USERNAME if not password: password = DEFAULT_CVM_PASSWD transport = Transport((ip, 22)) transport.connect(username=username, password=password) sftp = SFTPClient.from_transport(transport) sftp.get(remote_file_path, local_file_path)
def sftpOpenConnection(self, target): """ opens an sftp connection to the given target host; credentials are taken from /etc/ssh/sftp_passwd """ from paramiko import Transport, SFTPClient from paramiko.util import load_host_keys hostname, username, password = self.getCredentials(target) hostkeys = load_host_keys('/etc/ssh/ssh_known_hosts') hostkeytype, hostkey = hostkeys[hostname].items()[0] trans = Transport((hostname, 22)) trans.connect(username=username, password=password, hostkey=hostkey) return SFTPClient.from_transport(trans)
def connect(self): if not self.session_ and not self.transport_.is_active(): if self.key_: # Try private key, if available self.transport_.connect(username=self.user_, pkey=self.key_) elif len(self.pwd_) > 0: # Next, try password, if available self.transport_.connect(username=self.user_, password=self.pwd_) else: # Try just the user name self.transport_.connect(username=self.user_) self.session_ = SFTPClient.from_transport(self.transport_)
def __enter__(self): self.set_transport() self.sftp = SFTPClient.from_transport(self.transport) try: logger.info("Try to make a directory") self.sftp.mkdir(self.remote_dir) except Exception as e: logger.info(e) logger.info(f"Change the current directory into {self.remote_dir}") self.sftp.chdir(self.remote_dir) return self.sftp
def _get_sftp_connection(host): # SFTP connections aren't *required* for deploys, so we create them on-demand if host in SFTP_CONNECTIONS: return SFTP_CONNECTIONS[host] transport = host.connection.get_transport() client = SFTPClient.from_transport(transport) SFTP_CONNECTIONS[host] = client return client
def run_scp(host_info, src, dest): """ connect to host and send srouce file to destination """ try: ssh_client = SSHClient() ssh_client.set_missing_host_key_policy(AutoAddPolicy()) ssh_client.connect(host_info.ip, host_info.port, host_info.user, key_filename=host_info.key) sftp_client = SFTPClient.from_transport(ssh_client.get_transport()) sftp_client.put(src, dest) sftp_client.close() except Exception as e: raise e
def _connect(self): self._conn = Transport((self._host, self._port)) self._conn.window_size = pow(2, 27) self._conn.packetizer.REKEY_BYTES = pow(2, 32) self._conn.packetizer.REKEY_PACKETS = pow(2, 32) if self._auth == 'key': pkey = RSAKey.from_private_key_file(self._key_path) self._conn.connect(username=self._user, pkey=pkey) else: self._conn.connect(username=self._user, password=self._passwd) self._client = SFTPClient.from_transport(self._conn)
def release_success(self, build_info): def progress(filename, transferred, total): print( colorize('fuchsia', ' * %s transfer in progress: %02d%%.\r' % (filename, transferred * 100 / total)), end='\r' ) ssh = SSHClient() try: ssh.set_missing_host_key_policy(AutoAddPolicy()) ssh.connect(self.ssh['server'], username=self.ssh['user'], key_filename=self.ssh['pkey']) except SSHException: raise XUtilsError('Unable to establish a SSH connection to %s' % self.ssh['server']) dest_dir = '/'.join([ self.ssh['base_dir'], build_info['category'], build_info['pkg_name'], build_info['version'], build_info['arch'] ]) stdin, stdout, stderr = ssh.exec_command('mkdir -p %s' % dest_dir) # pylint: disable=unused-variable if stderr.read(): raise XUtilsError('Unable to create directory %s on server %s' % (dest_dir, self.ssh['server'])) src_dir = str() for f in listdir(self.cfg['build']['workdir']): if f.endswith('_root.tar.gz'): src_dir = self.cfg['build']['workdir'] if not src_dir: src_dir = self.cfg['release']['archive_dir'] files = [ '%s-%s_root.tar.gz' % (build_info['pkg_name'], build_info['version']), '%s-%s_debuginfo.tar.gz' % (build_info['pkg_name'], build_info['version']), '%s-%s_root.tar.gz.gpg' % (build_info['pkg_name'], build_info['version']), ] try: sftp = SFTPClient.from_transport(ssh.get_transport()) sftp.chdir(dest_dir) except SSHException: raise XUtilsError('Unable to negotiate a SFTP session for %s' % self.ssh['server']) for f in files: filepath = realpath(src_dir + '/' + f) if not exists(filepath): self.info(colorize('yellow', '%s not found => skip it.' % f)) continue my_callback = partial(progress, f) remote_attr = sftp.put(filepath, f, callback=my_callback) if stat(filepath).st_size == remote_attr.st_size: self.info('%s successfully copied on server %s.' % (f, self.ssh['server'])) else: raise XUtilsError('Copy of %s on server %s is corrupted.' % (f, self.ssh['server'])) sftp.close() ssh.close()
def _connect(self): if self._conn is not None: log.warn("Already connected to sftp server") return try: self._conn = Transport((self._host, self._port)) self._conn.window_size = pow(2, 27) self._conn.packetizer.REKEY_BYTES = pow(2, 32) self._conn.packetizer.REKEY_PACKETS = pow(2, 32) self._conn.connect(username=self._user, password=self._passwd) self._client = SFTPClient.from_transport(self._conn) except Exception as e: raise IrmaSFTPError("{0}".format(e))
def sftpOpen(self, target): """ opens an sftp connection to the given target host; credentials are taken from /etc/ssh/sftp_passwd """ from paramiko import Transport, SFTPClient #from paramiko.util import load_host_keys print '>>> sftpOpen self.username: '******' self.password: '******' self.hostkey: ', self.password print ' self.hostname: ', self.hostname trans = Transport((self.hostname, 22)) trans.banner_timeout = 120 trans.connect(username=self.username, password=self.password, hostkey=self.hostkey) return SFTPClient.from_transport(trans)
def _putFile(self, data, connection): """ Puts the given file on the TSM host. """ sftp = SFTPClient.from_transport(connection) try: try: remoteFileObject = sftp.open(self._persistenceIdentifier, "w") block = data.read(_BLOCK_SIZE) while len(block) > 0: remoteFileObject.write(block) block = data.read(_BLOCK_SIZE) except (IOError, SSHException), sshException: errorMessage = "Cannot transfer data to TSM host!\nReason: '%s'" % str(sshException) raise PersistenceError(errorMessage) finally: data.close() sftp.close()
def listdir(hostname, path="/var/tmp", filter="", port=1035, username="", password=""): """ paramiko sftp listdir wrapper, with option to filter files """ # Paramiko client configuration t = Transport((hostname, port)) t.connect(username=username, password=password) sftp = SFTPClient.from_transport(t) try: rex = re.compile(filter) except: print "Invalid regular expression: " + filter sys.exit(1) return [x for x in sftp.listdir(path) if rex.match(x)]
def upload_report_to_sftp(self, client_id, report_date, absolute_filename): """ Upload the given file, using SFTP, to the configured FTP server. The file should be uploaded to the appropriate directory for the specified client and the date of the report. """ try: client = Clients.objects.get(id=client_id) except Clients.DoesNotExist: logger.exception(u'No configuration for client {0}.'.format(client_id)) raise filename = basename(absolute_filename) base_folder, env_folder, year_folder, month_folder = self._get_sftp_dirs(client, report_date) try: logger.debug(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME)) transport = Transport((settings.SFTP_SERVER, settings.SFTP_PORT)) transport.connect(username=settings.SFTP_USERNAME, password=settings.SFTP_PASSWORD) sftp = SFTPClient.from_transport(transport) logger.debug(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder)) sftp.chdir(base_folder) self._make_or_change_sftp_dir(sftp, env_folder) self._make_or_change_sftp_dir(sftp, year_folder) self._make_or_change_sftp_dir(sftp, month_folder) logger.debug(u'SFTP uploading {0}'.format(filename)) sftp.put(absolute_filename, filename) except Exception: logger.exception(u'Unrecoverable exception during SFTP upload process.') raise finally: logger.debug(u'SFTP logging off') try: sftp.close() except Exception: logger.exception(u'SFTP exception while closing SFTP session.') try: transport.close() except Exception: logger.exception(u'SFTP exception while closing SSH connection.')
def sftp_connect(config): try: hostname = config["hostname"] port = int(config["port"]) username = config["username"] pkeyfile = config["privatekey_file"] pkeypassword = config["privatekey_passphrase"] host_keys = load_host_keys(config["hostkeys_file"]) hostkeytype, hostkey = get_hostkeytype_and_hostkey(host_keys, config["hostname"]) t = Transport((hostname, port)) pkey = get_privatekey_from_file(pkeyfile, pkeypassword) t.connect(username=username, pkey=pkey, hostkey=hostkey) return SFTPClient.from_transport(t) except SSHException, e: print " [*] " + str(e) sys.exit(1)
def connect(self): """ Get a handle to a remote connection """ # Check URL schema = urlparse(self.url) if schema.scheme == 'sftp': self.transport = Transport((schema.hostname, int(schema.port))) else: raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme)) # Add authentication to transport try: if self.password: self.transport.connect(username=self.user_name, password=self.password) elif self.private_key: self.transport.connect(username=self.user_name, pkey=RSAKey.from_private_key(StringIO(self.private_key))) else: raise SFTPError("No password or private_key defined") # Connect self.conn = SFTPClient.from_transport(self.transport) except (socket.gaierror,error), msg: raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
def _getFile(self, connection): """ Transfers the given file from the TSM host to the local file system. """ try: temporaryFileObject = tempfile.TemporaryFile() sftp = SFTPClient.from_transport(connection) try: temporaryFileObject.seek(0) remoteFileObject = sftp.open(self._persistenceIdentifier) block = remoteFileObject.read(_BLOCK_SIZE) while len(block) > 0: temporaryFileObject.write(block) block = remoteFileObject.read(_BLOCK_SIZE) sftp.remove(self._persistenceIdentifier) temporaryFileObject.seek(0) return temporaryFileObject except (IOError, SSHException), error: errorMessage = "Cannot retrieve file from TSM host!\nReason: '%s'" % str(error) raise PersistenceError(errorMessage) finally: sftp.close()
def connect(self): """ Get a handle to a remote connection """ # Check URL schema = urlparse(self.url) if schema.scheme == 'sftp': hostname = schema.hostname port = int(schema.port) # Socket creation code inspired from paramiko.Transport.__init__ # with added bind support. for family, socktype, _, _, _ in getaddrinfo( hostname, port, AF_UNSPEC, SOCK_STREAM, ): if socktype == SOCK_STREAM: sock = socket(family, SOCK_STREAM) if self.bind_address: # XXX: Expects bind address to be of same family as hostname. # May not be easy if name resolution is involved. # Try to reconciliate them ? sock.bind((self.bind_address, 0)) retry_on_signal(lambda: sock.connect((hostname, port))) break else: raise SFTPError('No suitable socket family found') self.transport = Transport(sock) else: raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme)) # Add authentication to transport try: if self.password: self.transport.connect(username=self.user_name, password=self.password) elif self.private_key: self.transport.connect(username=self.user_name, pkey=RSAKey.from_private_key(StringIO(self.private_key))) else: raise SFTPError("No password or private_key defined") # Connect self.conn = SFTPClient.from_transport(self.transport) except (gaierror, error), msg: raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
def get_sftp_conn(config): """Make a SFTP connection, returns sftp client and connection objects""" remote = config.get('remote_location') parts = urlparse(remote) if ':' in parts.netloc: hostname, port = parts.netloc.split(':') else: hostname = parts.netloc port = 22 port = int(port) username = config.get('remote_username') or getuser() luser = get_local_user(username) sshdir = get_ssh_dir(config, luser) hostkey = get_host_keys(hostname, sshdir) try: sftp = None keys = get_ssh_keys(sshdir) transport = Transport((hostname, port)) while not keys.empty(): try: key = PKey.from_private_key_file(keys.get()) transport.connect( hostkey=hostkey, username=username, password=None, pkey=key) sftp = SFTPClient.from_transport(transport) break except (PasswordRequiredException, SSHException): pass if sftp is None: raise SaChannelUpdateTransportError("SFTP connection failed") return sftp, transport except BaseException as msg: raise SaChannelUpdateTransportError(msg)
def __call__(self): registry = getUtility(IRegistry) recensio_settings = registry.forInterface(IRecensioSettings) host = recensio_settings.xml_export_server username = recensio_settings.xml_export_username password = recensio_settings.xml_export_password if not host: return 'no host configured' log.info("Starting XML export to sftp") exporter = getUtility(IFactory, name='chronicon_exporter')() export_xml = exporter.get_export_obj(self.context) if export_xml is None: msg = "Could not get export file object: {0}".format( exporter.export_filename) log.error(msg) return msg zipstream = export_xml.getFile() try: transport = Transport((host, 22)) transport.connect(username=username, password=password) sftp = SFTPClient.from_transport(transport) attribs = sftp.putfo(zipstream.getBlob().open(), self.filename) except (IOError, SSHException) as ioe: msg = "Export failed, {0}: {1}".format(ioe.__class__.__name__, ioe) log.error(msg) return msg if attribs.st_size == zipstream.get_size(): msg = "Export successful" log.info(msg) return msg else: msg = "Export failed, {0}/{1} bytes transferred".format( attribs.st_size, zipstream.get_size()) log.error(msg) return msg
def upload_report(self, filepath, file_name, report_date): """ >>> a = CSVgenerator('123') >>> a.upload_report(None, 'upload.txt', '2014-05-06') """ # appropriate changes required for doc test to run - dev only. # test_file_path = join(settings.REPORTS_ROOT, '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv') # # trace.info('----- {}'.format(abspath(test_file_path))) # absolute_filename = abspath(test_file_path) # test_filename = '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv' #TODO remove after QA testing. if not filepath: test_file_path = join(settings.REPORTS_ROOT, file_name) absolute_filename = abspath(test_file_path) else: absolute_filename = abspath(filepath) try: report_date = report_date if isinstance(report_date, str) else report_date.strftime(settings.DATE_FORMAT_YMD) except Exception: log.exception('') try: #TODO use sftp credentials, once available. #TODO get client upload location for per-client report. # ftp_client_dir = client.ftp_client_dir # if ftp_client_dir == '': # ftp_client_dir = getattr(settings, 'DEFAULT_SFTP_LOCATION', 'default') # logger.exception(u'No FTP configuration for client {} using default value.'.format(client.name)) # year_folder, month_folder, _ = report_date.split('-') base_folder_path = self._sftp_settings.get('path', '') base_path = base_folder_path.split('/') base_folders = [i for i in base_path[1:-1]] base_folder = '/' + join(*base_folders) env_folder = str(base_path[-1:][0]) from paramiko import Transport, SFTPClient try: log.info(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME)) transport = Transport(( self._sftp_settings.get('server', ''), self._sftp_settings.get('port', 22)) ) transport.connect( username=self._sftp_settings.get('username', ''), password=self._sftp_settings.get('password', '') ) sftp = SFTPClient.from_transport(transport) log.info(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder)) try: sftp.chdir(base_folder) except Exception: log.debug('Unable to change to base folder on ftp server.') self._make_or_change_sftp_dir(sftp, env_folder) self._make_or_change_sftp_dir(sftp, year_folder) self._make_or_change_sftp_dir(sftp, month_folder) log.debug(u'SFTP uploading {0}'.format(filepath)) sftp.put(absolute_filename, file_name) except Exception: log.exception(u'Unrecoverable exception during SFTP upload process.') finally: log.debug(u'SFTP logging off') try: sftp.close() except Exception: log.exception(u'SFTP exception while closing SFTP session.') try: transport.close() except Exception: log.exception(u'SFTP exception while closing SSH connection.') except Exception: log.debug('Error while uploading report.')
# -*- coding: utf-8 -*- __author__ = 'Administrator' from paramiko import Transport,RSAKey,SFTPClient import os host="54.187.91.18" port=22 private_key_file=os.path.expanduser("~/.ssh/id_rsa") key=RSAKey.from_private_key_file(private_key_file) t=Transport((host,port)) t.connect(username="******",pkey=key) sftp=SFTPClient.from_transport(t) try: sftp.put("/data/websites/test/a.php","/data/websites/") except Exception,e: print(e) sftp.close() t.close()
def copy(self, local_file, remote_file): if not hasattr(self, 'ssh_connection'): self.__get_ssh_connection() sftp = SFTPClient.from_transport(self.ssh_connection.get_transport()) sftp.put(local_file, remote_file)
def submitJobToFramework(self, **kwargs): jobCommand = 'job' daemonArgs = DaemonArgs(self.config) daemonArgs.command = jobCommand unScheduledJob = kwargs['unScheduledJob'] is_fileFeeder = False fileFeederUploadedFile = None del daemonArgs.param[:] # go through all parameters for parameter in unScheduledJob.parameters.all(): # add parameter to daemonArgs.param if parameter.service and parameter.param_key and parameter.param_value: # check if a file feeder is used if parameter.service == settings.FILE_FEEDER_ID: is_fileFeeder = True fileFeederUploadedFile = parameter.param_value remoteFeederFile = os.path.join(self.sftpRemotePath, parameter.param_value) parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, remoteFeederFile ) else: parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, parameter.param_value ) self.logger.debug("add parameter string: %s" % parameterString) daemonArgs.param.append([parameterString]) # in case of a filefeeder upload file to framework server if is_fileFeeder: self.logger.debug("is file feeder") sftp = None transport = None try: transport = Transport((self.sftpHost, self.sftpPort)) if self.sftpPassword: transport.connect(username=self.sftpUsername, password=self.sftpPassword) else: privateKey = None if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'rsa': privateKey = RSAKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword ) if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'dss': privateKey = DSSKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword ) transport.connect(username=self.sftpUsername, pkey=privateKey) sftp = SFTPClient.from_transport(transport) filePath = os.path.join( settings.MEDIA_ROOT, fileFeederUploadedFile ) remotePath = os.path.join( self.sftpRemotePath, fileFeederUploadedFile ) self.logger.debug("uploading file from %s to %s on remote machine" % (filePath, remotePath)) sftp.put(filePath, remotePath) # sftp.put(filePath, remotePath, confirm=False) sftp.chmod( remotePath, 0644 ) self.logger.debug("put OK") except IOError as e: self.logger.error("IOError: %s. Will continue with next scheduled job." % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) except PasswordRequiredException as e: self.logger.error("PasswordRequiredException: %s. Will continue with next scheduled job." % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) except SSHException as e: self.logger.error("SSH Exception: %s. Will continue with next scheduled job." % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) except Exception as e: self.logger.error("Unkown SFTP problem. Will continue with next scheduled job. %s" % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) finally: if sftp is not None: sftp.close() if transport is not None: transport.close() # set job workflow daemonArgs.jd_workflow = unScheduledJob.workflow.name frameworkJobId = None try: setattr(daemonArgs, jobCommand, 'submit') frameworkJobId = self.sendFrameworkCommand(jobCommand, daemonArgs) self.saveJob(Job.PROCESSING_STATUS, frameworkJobId, unScheduledJob) except WorkflowNotDeployedException: # The workflow is not deployed in the framework. To prevent the scheduler retrying continuously # we disable this job unScheduledJob.status = Schedule.DEACTIVATE_STATUS unScheduledJob.save() except: self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) finally: daemonArgs.clean(jobCommand) if unScheduledJob.scheduled_start is not None: unScheduledJob.status = Schedule.DEACTIVATED_STATUS unScheduledJob.save()
def Link(self,\ sftp=None,\ ssh=True): self.sock = socket() self.sock.settimeout(self.timeout) self.Errors = [] self.protocol = [] self.logger.info('Trying to start link using parameters [SSH=%s,SFTP=%s]'%(ssh,sftp)) try: self.sock.connect((self.remoteMachine, self.port)) except: self.Unlink() self.logger.error('Link: Could not open a socket [remoteMachine = %s, port = %s]' % (self.remoteMachine, self.port)) self.transport = Transport(self.sock) try: if not self.password: self.transport.connect(username = self.username, pkey = self.pk) self.logger.info('Trying to connect using private key..') else: self.transport.connect(username = self.username, password = self.password) self.logger.info('Trying to connect using password..') except: if not self.password: self.logger.error('Private Key not installed and no password specified!') self.Unlink() self.logger.error('Could not connect using SFTP [remoteMachine = %s, Username = %s, port = %s]' %(self.remoteMachine, self.username, self.port)) sftp=None if ssh: try: self.protocol.append(mySSHClient()) self.ssh = self.protocol[-1] self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.ssh.connect(self.remoteMachine, port = self.port, username = self.username, password = self.password) self.logger.info ('ssh connection created..') except: self.logger.error('Could not connect using SSH [remoteMachine = %s, Username = %s, port = %s]'%(self.remoteMachine, self.username, self.port)) if sftp: try: if ssh == None: self.logger.error ('SSH must be enabled in order to connect with SFTP') self.protocol.append(SFTPClient.from_transport(self.transport)) self.sftp = self.protocol[-1] self.client = self.sftp self.logger.info('sftp connection created..') except: self.logger.error ('Could not bring up SFTP connection') self.sftp=None return ssh, sftp
def connect(self): self._transport = Transport((self._host, self._port)) self._transport.connect(username=self._user, password=self._pass) self._client = SFTPClient.from_transport(self._transport)
def __connect(self): self.t = Transport((self.config['hostname'], self.config['port'])) self.t.connect(username = self.config['user'], pkey = self.auth_key) self.client = SFTPClient.from_transport(self.t) self.client.chdir(self.path)
def connect(self, host, port=22, user=None, passw=None, cert=None, path='/', timeout=10): """Method connects to server Args: host (str): server host port (int): server port, default protocol port user (str): username passw (str): password cert (str): path to certificate file path (str): server path timeout (int): timeout Returns: bool: result Raises: event: ftp_before_connect event: ftp_after_connect """ try: message = '{0}/{1}@{2}:{3}{4} cert:{5}, timeout:{6}'.format( user, passw, host, port, path, cert, timeout) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_connecting', message), self._mh.fromhere()) ev = event.Event( 'ftp_before_connect', host, port, user, passw, cert, path, timeout) if (self._mh.fire_event(ev) > 0): host = ev.argv(0) port = ev.argv(1) user = ev.argv(2) passw = ev.argv(3) cert = ev.argv(4) path = ev.argv(5) timeout = ev.argv(6) self._host = host self._port = port self._user = user self._passw = passw self._cert = cert if (ev.will_run_default()): setdefaulttimeout(timeout) t = Transport((host, self._port)) if (user != None or cert != None): pkey = RSAKey.from_private_key_file( self._cert) if (cert != None) else None t.connect(username=user, password=passw, pkey=pkey) self._client = SFTPClient.from_transport(t) self._is_connected = True if (path != None): self.change_dir(path) ev = event.Event('ftp_after_connect') self._mh.fire_event(ev) return True except (SSHException, NoValidConnectionsError, error) as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False