def execute(self, client, stage_number): """ Move the tar file into place, unpack and run bootstrap.sh passing any given arguments. """ sftp = SFTPClient.from_transport(client.get_transport()) stage_directory = 'stage-' + str(stage_number) try: sftp.mkdir(stage_directory) except: logger.exception('') sftp.chdir(stage_directory) # Stage complete so nothing to do. if 'stage-complete' in sftp.listdir(): return # upload, untar, execute bootstrap.sh sftp.put(self.tarfile, '{0}/{1}.tar'.format(sftp.getcwd(), stage_directory)) untar = 'cd {0} && tar xf {1}.tar'.format(stage_directory, stage_directory) untar_result = self.execute_command(untar, client) logger.debug('Command results: {0}.'.format(untar_result)) logger.info('Executing contents of tar file: {0}.'.format(self.tarfile)) bootstrap_arguments = ' '.join(self.args) run_bootstrap = 'cd {0} '.format( stage_directory) + '&& sudo -u root -H bash -l -c "bash bootstrap.sh {0}" > output '.format( bootstrap_arguments) + '&& touch stage-complete' bootstrap_result = self.execute_command(run_bootstrap, client) if bootstrap_result[1] != 0: logger.error("Stage did not complete: {0}.".format(stage_number)) logger.debug('Command results: {0}.'.format(bootstrap_result[0]))
def _copy_to_server(self, filename, destination=None): self._ensure_connected() if destination is None: destination = self.working_directory sftp = SFTPClient.from_transport(self.ssh.get_transport()) # tar and compress a directory before copying if os.path.isfile(filename): sftp.put(filename, os.path.join(destination, os.path.basename(filename))) elif os.path.isdir(filename): tar_filename = filename + '_' + str(uuid.uuid4()) + '.tar.gz' self.logger.info("Compressing " + filename + '...') subprocess.call(['tar', 'cfz', tar_filename, filename]) self.logger.info("Copying " + tar_filename + " to " + self.hostname + ':' + os.path.join(destination, os.path.basename(tar_filename)) + '...') sftp.put(tar_filename, os.path.join(destination, os.path.basename(tar_filename))) os.remove(tar_filename) _, stdout, stderr = self._run_command('cd ' + quote(self.working_directory) + ' && tar xvfz ' + quote(tar_filename) + ' && rm ' + quote(tar_filename)) self.logger.debug("tar STDOUT: " + stdout.read()) self.logger.debug("tar STDERR: " + stderr.read()) else: raise NotImplementedError("Can't SFTP put anything other than a folder or file yet.") sftp.close()
def write_redirects_to_sftp(self, from_path, to_path, cron): try: ssh_key_object = RSAKey(filename=app.config['SFTP_SSH_KEY_PATH'], password=app.config['SFTP_SSH_KEY_PASSPHRASE']) remote_server_public_key = HostKeyEntry.from_line(app.config['SFTP_REMOTE_HOST_PUBLIC_KEY']).key # This will throw a warning, but the (string, int) tuple will automatically be parsed into a Socket object remote_server = Transport((app.config['SFTP_REMOTE_HOST'], 22)) remote_server.connect(hostkey=remote_server_public_key, username=app.config['SFTP_USERNAME'], pkey=ssh_key_object) sftp = SFTPClient.from_transport(remote_server) sftp.put(from_path, to_path) if cron: return 'SFTP publish from %s to %s succeeded' % (from_path, to_path) else: return fjson.dumps({ 'type': 'success', 'message': 'Redirect updates successful' }) except: if cron: return 'SFTP publish from %s to %s failed' % (from_path, to_path) else: return fjson.dumps({ 'type': 'danger', 'message': 'Redirect updates failed' })
def upload(dest, srcPath, destPath, callBack, skipIfExists = False): try: sftp = SFTPClient.from_transport(dest.connection) # This is pretty awful, if the file doesn't exists it throws # an exception, and then we should continue # Assume true exists = True if skipIfExists: try: sftp.stat(destPath) except FileNotFoundError as e: exists = False if not skipIfExists or not exists: renameUpload(sftp, srcPath, destPath, callBack) else: callBack() return True except Exception as e: print('*** Caught exception: %s: %s' % (e.__class__, e)) traceback.print_exc() sys.exit(1) finally: try: t.close() except: pass return False
def _authenticate(self): self._transport = SFTPTransport((self.config['sftp_host'], self.config['sftp_port'])) self._transport.connect(username=self.config['sftp_user'], password=self.config['sftp_password']) self.session = SFTPClient.from_transport(self._transport) logging.info('SFTP Authorization succeed')
def scp(self, srcFile, destPath): transport = Transport((self.host, int(self.port))) transport.connect(username=self.user, password=self.passwd) sftp = SFTPClient.from_transport(transport) try: sftp.put(srcFile, destPath) except IOError as e: raise e
def __init__(self, host, port, username, password): self.host = host self.port = port self.username = username self.password = password self.t = Transport((self.host, self.port)) self.t.connect(username=self.username, password=self.password) # 注意:此处需要用默认传参方式传参 self.sftp = SFTPClient.from_transport(self.t)
def copy(self, local_file, remote_file): if self.localMachine: command = "cp %s %s" % (local_file, remote_file) return self.execute_command(command) else: if not hasattr(self, 'ssh_connection'): self.__get_ssh_connection() sftp = SFTPClient.from_transport(self.ssh_connection.get_transport()) obj = sftp.put(local_file, remote_file) return (0, "Copied to %s" % obj.__str__(), "")
def sftpOpenConnection(self, target): """ opens an sftp connection to the given target host; credentials are taken from /etc/ssh/sftp_passwd """ from paramiko import Transport, SFTPClient from paramiko.util import load_host_keys hostname, username, password = self.getCredentials(target) hostkeys = load_host_keys('/etc/ssh/ssh_known_hosts') hostkeytype, hostkey = hostkeys[hostname].items()[0] trans = Transport((hostname, 22)) trans.connect(username=username, password=password, hostkey=hostkey) return SFTPClient.from_transport(trans)
def release_success(self, build_info): def progress(filename, transferred, total): print( colorize('fuchsia', ' * %s transfer in progress: %02d%%.\r' % (filename, transferred * 100 / total)), end='\r' ) ssh = SSHClient() try: ssh.set_missing_host_key_policy(AutoAddPolicy()) ssh.connect(self.ssh['server'], username=self.ssh['user'], key_filename=self.ssh['pkey']) except SSHException: raise XUtilsError('Unable to establish a SSH connection to %s' % self.ssh['server']) dest_dir = '/'.join([ self.ssh['base_dir'], build_info['category'], build_info['pkg_name'], build_info['version'], build_info['arch'] ]) stdin, stdout, stderr = ssh.exec_command('mkdir -p %s' % dest_dir) # pylint: disable=unused-variable if stderr.read(): raise XUtilsError('Unable to create directory %s on server %s' % (dest_dir, self.ssh['server'])) src_dir = str() for f in listdir(self.cfg['build']['workdir']): if f.endswith('_root.tar.gz'): src_dir = self.cfg['build']['workdir'] if not src_dir: src_dir = self.cfg['release']['archive_dir'] files = [ '%s-%s_root.tar.gz' % (build_info['pkg_name'], build_info['version']), '%s-%s_debuginfo.tar.gz' % (build_info['pkg_name'], build_info['version']), '%s-%s_root.tar.gz.gpg' % (build_info['pkg_name'], build_info['version']), ] try: sftp = SFTPClient.from_transport(ssh.get_transport()) sftp.chdir(dest_dir) except SSHException: raise XUtilsError('Unable to negotiate a SFTP session for %s' % self.ssh['server']) for f in files: filepath = realpath(src_dir + '/' + f) if not exists(filepath): self.info(colorize('yellow', '%s not found => skip it.' % f)) continue my_callback = partial(progress, f) remote_attr = sftp.put(filepath, f, callback=my_callback) if stat(filepath).st_size == remote_attr.st_size: self.info('%s successfully copied on server %s.' % (f, self.ssh['server'])) else: raise XUtilsError('Copy of %s on server %s is corrupted.' % (f, self.ssh['server'])) sftp.close() ssh.close()
def _connect(self): self._conn = Transport((self._host, self._port)) self._conn.window_size = pow(2, 27) self._conn.packetizer.REKEY_BYTES = pow(2, 32) self._conn.packetizer.REKEY_PACKETS = pow(2, 32) if self._auth == 'key': pkey = RSAKey.from_private_key_file(self._key_path) self._conn.connect(username=self._user, pkey=pkey) else: self._conn.connect(username=self._user, password=self._passwd) self._client = SFTPClient.from_transport(self._conn)
def _connect(self): if self._conn is not None: log.warn("Already connected to sftp server") return try: self._conn = Transport((self._host, self._port)) self._conn.window_size = pow(2, 27) self._conn.packetizer.REKEY_BYTES = pow(2, 32) self._conn.packetizer.REKEY_PACKETS = pow(2, 32) self._conn.connect(username=self._user, password=self._passwd) self._client = SFTPClient.from_transport(self._conn) except Exception as e: raise IrmaSFTPError("{0}".format(e))
def sftpOpen(self, target): """ opens an sftp connection to the given target host; credentials are taken from /etc/ssh/sftp_passwd """ from paramiko import Transport, SFTPClient #from paramiko.util import load_host_keys print '>>> sftpOpen self.username: '******' self.password: '******' self.hostkey: ', self.password print ' self.hostname: ', self.hostname trans = Transport((self.hostname, 22)) trans.banner_timeout = 120 trans.connect(username=self.username, password=self.password, hostkey=self.hostkey) return SFTPClient.from_transport(trans)
def _putFile(self, data, connection): """ Puts the given file on the TSM host. """ sftp = SFTPClient.from_transport(connection) try: try: remoteFileObject = sftp.open(self._persistenceIdentifier, "w") block = data.read(_BLOCK_SIZE) while len(block) > 0: remoteFileObject.write(block) block = data.read(_BLOCK_SIZE) except (IOError, SSHException), sshException: errorMessage = "Cannot transfer data to TSM host!\nReason: '%s'" % str(sshException) raise PersistenceError(errorMessage) finally: data.close() sftp.close()
def listdir(hostname, path="/var/tmp", filter="", port=1035, username="", password=""): """ paramiko sftp listdir wrapper, with option to filter files """ # Paramiko client configuration t = Transport((hostname, port)) t.connect(username=username, password=password) sftp = SFTPClient.from_transport(t) try: rex = re.compile(filter) except: print "Invalid regular expression: " + filter sys.exit(1) return [x for x in sftp.listdir(path) if rex.match(x)]
def upload_report_to_sftp(self, client_id, report_date, absolute_filename): """ Upload the given file, using SFTP, to the configured FTP server. The file should be uploaded to the appropriate directory for the specified client and the date of the report. """ try: client = Clients.objects.get(id=client_id) except Clients.DoesNotExist: logger.exception(u'No configuration for client {0}.'.format(client_id)) raise filename = basename(absolute_filename) base_folder, env_folder, year_folder, month_folder = self._get_sftp_dirs(client, report_date) try: logger.debug(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME)) transport = Transport((settings.SFTP_SERVER, settings.SFTP_PORT)) transport.connect(username=settings.SFTP_USERNAME, password=settings.SFTP_PASSWORD) sftp = SFTPClient.from_transport(transport) logger.debug(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder)) sftp.chdir(base_folder) self._make_or_change_sftp_dir(sftp, env_folder) self._make_or_change_sftp_dir(sftp, year_folder) self._make_or_change_sftp_dir(sftp, month_folder) logger.debug(u'SFTP uploading {0}'.format(filename)) sftp.put(absolute_filename, filename) except Exception: logger.exception(u'Unrecoverable exception during SFTP upload process.') raise finally: logger.debug(u'SFTP logging off') try: sftp.close() except Exception: logger.exception(u'SFTP exception while closing SFTP session.') try: transport.close() except Exception: logger.exception(u'SFTP exception while closing SSH connection.')
def sftp_connect(config): try: hostname = config["hostname"] port = int(config["port"]) username = config["username"] pkeyfile = config["privatekey_file"] pkeypassword = config["privatekey_passphrase"] host_keys = load_host_keys(config["hostkeys_file"]) hostkeytype, hostkey = get_hostkeytype_and_hostkey(host_keys, config["hostname"]) t = Transport((hostname, port)) pkey = get_privatekey_from_file(pkeyfile, pkeypassword) t.connect(username=username, pkey=pkey, hostkey=hostkey) return SFTPClient.from_transport(t) except SSHException, e: print " [*] " + str(e) sys.exit(1)
def _getFile(self, connection): """ Transfers the given file from the TSM host to the local file system. """ try: temporaryFileObject = tempfile.TemporaryFile() sftp = SFTPClient.from_transport(connection) try: temporaryFileObject.seek(0) remoteFileObject = sftp.open(self._persistenceIdentifier) block = remoteFileObject.read(_BLOCK_SIZE) while len(block) > 0: temporaryFileObject.write(block) block = remoteFileObject.read(_BLOCK_SIZE) sftp.remove(self._persistenceIdentifier) temporaryFileObject.seek(0) return temporaryFileObject except (IOError, SSHException), error: errorMessage = "Cannot retrieve file from TSM host!\nReason: '%s'" % str(error) raise PersistenceError(errorMessage) finally: sftp.close()
def connect(self): """ Get a handle to a remote connection """ # Check URL schema = urlparse(self.url) if schema.scheme == 'sftp': self.transport = Transport((schema.hostname, int(schema.port))) else: raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme)) # Add authentication to transport try: if self.password: self.transport.connect(username=self.user_name, password=self.password) elif self.private_key: self.transport.connect(username=self.user_name, pkey=RSAKey.from_private_key(StringIO(self.private_key))) else: raise SFTPError("No password or private_key defined") # Connect self.conn = SFTPClient.from_transport(self.transport) except (socket.gaierror,error), msg: raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
def connect(self): """ Get a handle to a remote connection """ # Check URL schema = urlparse(self.url) if schema.scheme == 'sftp': hostname = schema.hostname port = int(schema.port) # Socket creation code inspired from paramiko.Transport.__init__ # with added bind support. for family, socktype, _, _, _ in getaddrinfo( hostname, port, AF_UNSPEC, SOCK_STREAM, ): if socktype == SOCK_STREAM: sock = socket(family, SOCK_STREAM) if self.bind_address: # XXX: Expects bind address to be of same family as hostname. # May not be easy if name resolution is involved. # Try to reconciliate them ? sock.bind((self.bind_address, 0)) retry_on_signal(lambda: sock.connect((hostname, port))) break else: raise SFTPError('No suitable socket family found') self.transport = Transport(sock) else: raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme)) # Add authentication to transport try: if self.password: self.transport.connect(username=self.user_name, password=self.password) elif self.private_key: self.transport.connect(username=self.user_name, pkey=RSAKey.from_private_key(StringIO(self.private_key))) else: raise SFTPError("No password or private_key defined") # Connect self.conn = SFTPClient.from_transport(self.transport) except (gaierror, error), msg: raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
def get_sftp_conn(config): """Make a SFTP connection, returns sftp client and connection objects""" remote = config.get('remote_location') parts = urlparse(remote) if ':' in parts.netloc: hostname, port = parts.netloc.split(':') else: hostname = parts.netloc port = 22 port = int(port) username = config.get('remote_username') or getuser() luser = get_local_user(username) sshdir = get_ssh_dir(config, luser) hostkey = get_host_keys(hostname, sshdir) try: sftp = None keys = get_ssh_keys(sshdir) transport = Transport((hostname, port)) while not keys.empty(): try: key = PKey.from_private_key_file(keys.get()) transport.connect( hostkey=hostkey, username=username, password=None, pkey=key) sftp = SFTPClient.from_transport(transport) break except (PasswordRequiredException, SSHException): pass if sftp is None: raise SaChannelUpdateTransportError("SFTP connection failed") return sftp, transport except BaseException as msg: raise SaChannelUpdateTransportError(msg)
def __call__(self): registry = getUtility(IRegistry) recensio_settings = registry.forInterface(IRecensioSettings) host = recensio_settings.xml_export_server username = recensio_settings.xml_export_username password = recensio_settings.xml_export_password if not host: return 'no host configured' log.info("Starting XML export to sftp") exporter = getUtility(IFactory, name='chronicon_exporter')() export_xml = exporter.get_export_obj(self.context) if export_xml is None: msg = "Could not get export file object: {0}".format( exporter.export_filename) log.error(msg) return msg zipstream = export_xml.getFile() try: transport = Transport((host, 22)) transport.connect(username=username, password=password) sftp = SFTPClient.from_transport(transport) attribs = sftp.putfo(zipstream.getBlob().open(), self.filename) except (IOError, SSHException) as ioe: msg = "Export failed, {0}: {1}".format(ioe.__class__.__name__, ioe) log.error(msg) return msg if attribs.st_size == zipstream.get_size(): msg = "Export successful" log.info(msg) return msg else: msg = "Export failed, {0}/{1} bytes transferred".format( attribs.st_size, zipstream.get_size()) log.error(msg) return msg
# -*- coding: utf-8 -*- __author__ = 'Administrator' from paramiko import Transport,RSAKey,SFTPClient import os host="54.187.91.18" port=22 private_key_file=os.path.expanduser("~/.ssh/id_rsa") key=RSAKey.from_private_key_file(private_key_file) t=Transport((host,port)) t.connect(username="******",pkey=key) sftp=SFTPClient.from_transport(t) try: sftp.put("/data/websites/test/a.php","/data/websites/") except Exception,e: print(e) sftp.close() t.close()
def Link(self,\ sftp=None,\ ssh=True): self.sock = socket() self.sock.settimeout(self.timeout) self.Errors = [] self.protocol = [] self.logger.info('Trying to start link using parameters [SSH=%s,SFTP=%s]'%(ssh,sftp)) try: self.sock.connect((self.remoteMachine, self.port)) except: self.Unlink() self.logger.error('Link: Could not open a socket [remoteMachine = %s, port = %s]' % (self.remoteMachine, self.port)) self.transport = Transport(self.sock) try: if not self.password: self.transport.connect(username = self.username, pkey = self.pk) self.logger.info('Trying to connect using private key..') else: self.transport.connect(username = self.username, password = self.password) self.logger.info('Trying to connect using password..') except: if not self.password: self.logger.error('Private Key not installed and no password specified!') self.Unlink() self.logger.error('Could not connect using SFTP [remoteMachine = %s, Username = %s, port = %s]' %(self.remoteMachine, self.username, self.port)) sftp=None if ssh: try: self.protocol.append(mySSHClient()) self.ssh = self.protocol[-1] self.ssh.set_missing_host_key_policy(AutoAddPolicy()) self.ssh.connect(self.remoteMachine, port = self.port, username = self.username, password = self.password) self.logger.info ('ssh connection created..') except: self.logger.error('Could not connect using SSH [remoteMachine = %s, Username = %s, port = %s]'%(self.remoteMachine, self.username, self.port)) if sftp: try: if ssh == None: self.logger.error ('SSH must be enabled in order to connect with SFTP') self.protocol.append(SFTPClient.from_transport(self.transport)) self.sftp = self.protocol[-1] self.client = self.sftp self.logger.info('sftp connection created..') except: self.logger.error ('Could not bring up SFTP connection') self.sftp=None return ssh, sftp
def upload_sftp(server_info, upload_files, delete_files, force_reupload=False): server, info = server_info t = Transport((info['host'], info['port'])) try: t.connect(username=info['username'], password=info['password']) except AuthenticationException as e: print('\t\t%s' % e) return False, [], [] plugins_load = [] plugins_reload = [] sftp = SFTPClient.from_transport(t) try: sftp.chdir(info['path']) except FileNotFoundError as e: print('\t\tCannot chdir to remote path %s: %s' % (info['path'], e)) return False, [], [] if delete_files: print('\t\tProcessing %d deletions...' % (len(delete_files))) for i, d in enumerate(delete_files): print('\t\t\t%d. %s' % (i+1, d)) try: file_info = sftp.stat(info['path'] + '/' + d) if stat.S_ISDIR(file_info.st_mode): sftp.rmdir(info['path'] + '/' + d) else: sftp.remove(info['path'] + '/' + d) except FileNotFoundError: pass except Exception as e: print('\t\t\t\tFailed: %s' % e) for (path, file) in upload_files: abs_path = info['path'] + '/' + path relative_filepath = path + '/' + file print('\t\tProcessing %s' % file) try: sftp.stat(abs_path) except FileNotFoundError: print('\t\t\t%s does not exist, creating...' % path) i=1 while i != -1: i = path.find('/', i) abs_path = info['path'] + path[0:i] if i != -1: try: sftp.stat(abs_path) i += 1 except FileNotFoundError: sftp.mkdir(abs_path) sftp.mkdir(info['path'] + path) local_filepath = 'upload' + relative_filepath try: fileinfo = sftp.stat(info['path'] + relative_filepath) print('\t\t\tRemote file found') upload = False if (force_reupload): print('\t\t\tForcing reupload') upload = True else: lsize = os.path.getsize(local_filepath) rsize = fileinfo.st_size if lsize != rsize: print('\t\t\tSize mismatch -- Reuploading') upload = True if upload: sftp.put(local_filepath, info['path'] + relative_filepath) if (file[-4:] == '.smx'): plugins_reload.append(file[:-4]) except FileNotFoundError: print('\t\t\tUploading') sftp.put(local_filepath, info['path'] + relative_filepath) if (file[-4:] == '.smx'): plugins_load.append(file[:-4]) t.close() print('\t\tLogout') return True, plugins_load, plugins_reload
async def pull_file(ftp: paramiko.SFTPClient, ftp_path: str, local_path: str): """Pulls file from remote path to local path only if out of date.""" if compare_files(ftp, ftp_path, local_path): ftp.get(ftp_path, local_path) else: print("Local is up to date.")
def submitJobToFramework(self, **kwargs): jobCommand = 'job' daemonArgs = DaemonArgs(self.config) daemonArgs.command = jobCommand unScheduledJob = kwargs['unScheduledJob'] is_fileFeeder = False fileFeederUploadedFile = None del daemonArgs.param[:] # go through all parameters for parameter in unScheduledJob.parameters.all(): # add parameter to daemonArgs.param if parameter.service and parameter.param_key and parameter.param_value: # check if a file feeder is used if parameter.service == settings.FILE_FEEDER_ID: is_fileFeeder = True fileFeederUploadedFile = parameter.param_value remoteFeederFile = os.path.join(self.sftpRemotePath, parameter.param_value) parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, remoteFeederFile ) else: parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, parameter.param_value ) self.logger.debug("add parameter string: %s" % parameterString) daemonArgs.param.append([parameterString]) # in case of a filefeeder upload file to framework server if is_fileFeeder: self.logger.debug("is file feeder") sftp = None transport = None try: transport = Transport((self.sftpHost, self.sftpPort)) if self.sftpPassword: transport.connect(username=self.sftpUsername, password=self.sftpPassword) else: privateKey = None if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'rsa': privateKey = RSAKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword ) if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'dss': privateKey = DSSKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword ) transport.connect(username=self.sftpUsername, pkey=privateKey) sftp = SFTPClient.from_transport(transport) filePath = os.path.join( settings.MEDIA_ROOT, fileFeederUploadedFile ) remotePath = os.path.join( self.sftpRemotePath, fileFeederUploadedFile ) self.logger.debug("uploading file from %s to %s on remote machine" % (filePath, remotePath)) sftp.put(filePath, remotePath) # sftp.put(filePath, remotePath, confirm=False) sftp.chmod( remotePath, 0644 ) self.logger.debug("put OK") except IOError as e: self.logger.error("IOError: %s. Will continue with next scheduled job." % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) except PasswordRequiredException as e: self.logger.error("PasswordRequiredException: %s. Will continue with next scheduled job." % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) except SSHException as e: self.logger.error("SSH Exception: %s. Will continue with next scheduled job." % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) except Exception as e: self.logger.error("Unkown SFTP problem. Will continue with next scheduled job. %s" % e) self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) finally: if sftp is not None: sftp.close() if transport is not None: transport.close() # set job workflow daemonArgs.jd_workflow = unScheduledJob.workflow.name frameworkJobId = None try: setattr(daemonArgs, jobCommand, 'submit') frameworkJobId = self.sendFrameworkCommand(jobCommand, daemonArgs) self.saveJob(Job.PROCESSING_STATUS, frameworkJobId, unScheduledJob) except WorkflowNotDeployedException: # The workflow is not deployed in the framework. To prevent the scheduler retrying continuously # we disable this job unScheduledJob.status = Schedule.DEACTIVATE_STATUS unScheduledJob.save() except: self.saveJob(Job.FAILED_STATUS, None, unScheduledJob) finally: daemonArgs.clean(jobCommand) if unScheduledJob.scheduled_start is not None: unScheduledJob.status = Schedule.DEACTIVATED_STATUS unScheduledJob.save()
def __upload( sftp_client: paramiko.SFTPClient, filename: str, local_path: str, remote_path: str ): with open(os.path.join(local_path, filename), "rb") as f: data = f.read() sftp_client.open(os.path.join(remote_path, filename), "wb").write(data)
def connect(self, host, port=22, user=None, passw=None, cert=None, path='/', timeout=10): """Method connects to server Args: host (str): server host port (int): server port, default protocol port user (str): username passw (str): password cert (str): path to certificate file path (str): server path timeout (int): timeout Returns: bool: result Raises: event: ftp_before_connect event: ftp_after_connect """ try: message = '{0}/{1}@{2}:{3}{4} cert:{5}, timeout:{6}'.format( user, passw, host, port, path, cert, timeout) self._mh.demsg('htk_on_debug_info', self._mh._trn.msg( 'htk_ftp_connecting', message), self._mh.fromhere()) ev = event.Event( 'ftp_before_connect', host, port, user, passw, cert, path, timeout) if (self._mh.fire_event(ev) > 0): host = ev.argv(0) port = ev.argv(1) user = ev.argv(2) passw = ev.argv(3) cert = ev.argv(4) path = ev.argv(5) timeout = ev.argv(6) self._host = host self._port = port self._user = user self._passw = passw self._cert = cert if (ev.will_run_default()): setdefaulttimeout(timeout) t = Transport((host, self._port)) if (user != None or cert != None): pkey = RSAKey.from_private_key_file( self._cert) if (cert != None) else None t.connect(username=user, password=passw, pkey=pkey) self._client = SFTPClient.from_transport(t) self._is_connected = True if (path != None): self.change_dir(path) ev = event.Event('ftp_after_connect') self._mh.fire_event(ev) return True except (SSHException, NoValidConnectionsError, error) as ex: self._mh.demsg( 'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere()) return False
def __connect(self): self.t = Transport((self.config['hostname'], self.config['port'])) self.t.connect(username = self.config['user'], pkey = self.auth_key) self.client = SFTPClient.from_transport(self.t) self.client.chdir(self.path)
def sftp_put_file(self, local_path, dest_path): self.t = Transport(sock=(self.host, self.ssh_port)) self.keyfile = self.__get_ssh_key() self.t.connect(username=self.user, pkey=self.keyfile) self.sftp = SFTPClient.from_transport(self.t) self.sftp.put(local_path, dest_path)
def _process_directory( self, level: int, sftp: paramiko.SFTPClient, remote_root: Path, remote_path: Path, local_targetdir: Path, entry: BackupEntry ): isroot = False options = entry.get_options() has_options = not is_empty_dict(options) path_rootindex = None recurse = False if has_options: if "recurse" in options and not self._check_option_ignored("recures"): recurse = options["recurse"] if "path_rootindex" in options and not self._check_option_ignored("path_rootindex"): path_rootindex = options["path_rootindex"] if len(remote_path.parents) == 0: isroot = True if isroot: localdir = local_targetdir else: if path_rootindex is not None and is_integer(path_rootindex): f = path_from_partindex(remote_path, path_rootindex) localdir = local_targetdir.joinpath(f) else: localdir = local_targetdir.joinpath(str(remote_path)[1:]) remote_filenode = None tabs2 = repeat("\t", level + 1) do_transfer = self._check_folder_with_options(remote_root, remote_path, options) if do_transfer is not None: self.info("{}Excluding '{}' due to json-folder-option {}".format( tabs2, remote_path, do_transfer )) return if not localdir.exists(): self.info("{}Creating parent folders '{}'".format(tabs2, localdir)) makedirs(str(localdir)) try: filelist = sftp.listdir(str(remote_path)) if len(filelist) > 0: for filenode in filelist: remote_filenode = remote_path.joinpath(filenode) remote_stat = sftp.lstat(str(remote_filenode)) if stat.S_ISDIR(remote_stat.st_mode): if recurse is True: self.info("\n{}Recursing into sub-directory '{}'".format( tabs2, remote_filenode )) self._process_directory( level + 1, sftp, remote_path, remote_filenode, local_targetdir, entry ) else: localfile = localdir.joinpath(filenode) self._download_file(level, sftp, remote_root, localfile, remote_filenode, entry) except Exception as e: if isinstance(e, PermissionError): self.info("{}PermissionError while listing contents of {}\n".format( tabs2, remote_filenode if remote_filenode is not None else remote_path )) elif isinstance(e, JobException): raise e else: raise JobException(e, 1)
def connect(self): self._transport = Transport((self._host, self._port)) self._transport.connect(username=self._user, password=self._pass) self._client = SFTPClient.from_transport(self._transport)
def _get_sftp_connection(host): transport = host.connection.get_transport() return SFTPClient.from_transport(transport)
def copy(self, local_file, remote_file): if not hasattr(self, 'ssh_connection'): self.__get_ssh_connection() sftp = SFTPClient.from_transport(self.ssh_connection.get_transport()) sftp.put(local_file, remote_file)
from traceback import format_exc import sys #import FEWSConnect import logging from paramiko import Transport from paramiko import SFTPClient if (__name__ == '__main__'): host = "ftp.ncep.noaa.gov" port = 22 #after much testing....port 22 is the place to be #usr = "******" #pw = "Shenandoah22" path = "/pub/data/nccf/com/nwm/prod/" #r"/Distribution/RHA-Data/rhapub5/hefs_icprb" lookback = 5 #ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/nwm/prod #https://ftp.ncep.noaa.gov/data/nccf/com/nwm/prod/ # try: # transport = Transport((host, port)) # transport.connect()#username = usr, password = pw) # sftp = SFTPClient.from_transport(transport) # print("1 it's alive...sftp object up and running") # except: # print("1 failed to connect") transport = Transport((host, port)) transport.connect() #username = usr, password = pw) sftp = SFTPClient.from_transport(transport)
def create_connection(cls, host, port, username, password): transport = Transport(sock=(host, port)) transport.connect(username=username, password=password) cls._connection = SFTPClient.from_transport(transport)
def __init__(self): self.transport = Transport(sock=(self.HOST, self.PORT)) self.transport.connect(username=self.USERNAME, password=self.PASSWORD) self.ftp_connection = SFTPClient.from_transport(self.transport)
def upload_report(self, filepath, file_name, report_date): """ >>> a = CSVgenerator('123') >>> a.upload_report(None, 'upload.txt', '2014-05-06') """ # appropriate changes required for doc test to run - dev only. # test_file_path = join(settings.REPORTS_ROOT, '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv') # # trace.info('----- {}'.format(abspath(test_file_path))) # absolute_filename = abspath(test_file_path) # test_filename = '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv' #TODO remove after QA testing. if not filepath: test_file_path = join(settings.REPORTS_ROOT, file_name) absolute_filename = abspath(test_file_path) else: absolute_filename = abspath(filepath) try: report_date = report_date if isinstance(report_date, str) else report_date.strftime(settings.DATE_FORMAT_YMD) except Exception: log.exception('') try: #TODO use sftp credentials, once available. #TODO get client upload location for per-client report. # ftp_client_dir = client.ftp_client_dir # if ftp_client_dir == '': # ftp_client_dir = getattr(settings, 'DEFAULT_SFTP_LOCATION', 'default') # logger.exception(u'No FTP configuration for client {} using default value.'.format(client.name)) # year_folder, month_folder, _ = report_date.split('-') base_folder_path = self._sftp_settings.get('path', '') base_path = base_folder_path.split('/') base_folders = [i for i in base_path[1:-1]] base_folder = '/' + join(*base_folders) env_folder = str(base_path[-1:][0]) from paramiko import Transport, SFTPClient try: log.info(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME)) transport = Transport(( self._sftp_settings.get('server', ''), self._sftp_settings.get('port', 22)) ) transport.connect( username=self._sftp_settings.get('username', ''), password=self._sftp_settings.get('password', '') ) sftp = SFTPClient.from_transport(transport) log.info(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder)) try: sftp.chdir(base_folder) except Exception: log.debug('Unable to change to base folder on ftp server.') self._make_or_change_sftp_dir(sftp, env_folder) self._make_or_change_sftp_dir(sftp, year_folder) self._make_or_change_sftp_dir(sftp, month_folder) log.debug(u'SFTP uploading {0}'.format(filepath)) sftp.put(absolute_filename, file_name) except Exception: log.exception(u'Unrecoverable exception during SFTP upload process.') finally: log.debug(u'SFTP logging off') try: sftp.close() except Exception: log.exception(u'SFTP exception while closing SFTP session.') try: transport.close() except Exception: log.exception(u'SFTP exception while closing SSH connection.') except Exception: log.debug('Error while uploading report.')