Example #1
1
    def execute(self, client, stage_number):
        """
        Move the tar file into place, unpack and run bootstrap.sh passing any given arguments.
        """
        sftp = SFTPClient.from_transport(client.get_transport())
        stage_directory = 'stage-' + str(stage_number)
        try:
            sftp.mkdir(stage_directory)
        except:
            logger.exception('')
        sftp.chdir(stage_directory)

        # Stage complete so nothing to do.
        if 'stage-complete' in sftp.listdir():
            return

        # upload, untar, execute bootstrap.sh
        sftp.put(self.tarfile, '{0}/{1}.tar'.format(sftp.getcwd(), stage_directory))
        untar = 'cd {0} && tar xf {1}.tar'.format(stage_directory, stage_directory)
        untar_result = self.execute_command(untar, client)
        logger.debug('Command results: {0}.'.format(untar_result))

        logger.info('Executing contents of tar file: {0}.'.format(self.tarfile))
        bootstrap_arguments = ' '.join(self.args)
        run_bootstrap = 'cd {0} '.format(
            stage_directory) + '&& sudo -u root -H bash -l -c "bash bootstrap.sh {0}" > output '.format(
            bootstrap_arguments) + '&& touch stage-complete'
        bootstrap_result = self.execute_command(run_bootstrap, client)
        if bootstrap_result[1] != 0:
            logger.error("Stage did not complete: {0}.".format(stage_number))

        logger.debug('Command results: {0}.'.format(bootstrap_result[0]))
Example #2
0
  def _copy_to_server(self, filename, destination=None):
    self._ensure_connected()

    if destination is None:
      destination = self.working_directory

    sftp = SFTPClient.from_transport(self.ssh.get_transport())

    # tar and compress a directory before copying
    if os.path.isfile(filename):
      sftp.put(filename, os.path.join(destination, os.path.basename(filename)))
    elif os.path.isdir(filename):
      tar_filename = filename + '_' + str(uuid.uuid4()) + '.tar.gz'

      self.logger.info("Compressing " + filename + '...')
      subprocess.call(['tar', 'cfz', tar_filename, filename])

      self.logger.info("Copying " + tar_filename + " to " + self.hostname + ':' + os.path.join(destination, os.path.basename(tar_filename)) + '...')
      sftp.put(tar_filename, os.path.join(destination, os.path.basename(tar_filename)))
      os.remove(tar_filename)

      _, stdout, stderr = self._run_command('cd ' + quote(self.working_directory) + ' && tar xvfz ' + quote(tar_filename) + ' && rm ' + quote(tar_filename))
      self.logger.debug("tar STDOUT: " + stdout.read())
      self.logger.debug("tar STDERR: " + stderr.read())
    else:
      raise NotImplementedError("Can't SFTP put anything other than a folder or file yet.")

    sftp.close()
    def write_redirects_to_sftp(self, from_path, to_path, cron):
        try:
            ssh_key_object = RSAKey(filename=app.config['SFTP_SSH_KEY_PATH'],
                                    password=app.config['SFTP_SSH_KEY_PASSPHRASE'])

            remote_server_public_key = HostKeyEntry.from_line(app.config['SFTP_REMOTE_HOST_PUBLIC_KEY']).key
            # This will throw a warning, but the (string, int) tuple will automatically be parsed into a Socket object
            remote_server = Transport((app.config['SFTP_REMOTE_HOST'], 22))
            remote_server.connect(hostkey=remote_server_public_key, username=app.config['SFTP_USERNAME'], pkey=ssh_key_object)

            sftp = SFTPClient.from_transport(remote_server)
            sftp.put(from_path, to_path)
            if cron:
                return 'SFTP publish from %s to %s succeeded' % (from_path, to_path)
            else:
                return fjson.dumps({
                    'type': 'success',
                    'message': 'Redirect updates successful'
                })
        except:
            if cron:
                return 'SFTP publish from %s to %s failed' % (from_path, to_path)
            else:
                return fjson.dumps({
                    'type': 'danger',
                    'message': 'Redirect updates failed'
                })
Example #4
0
def upload(dest, srcPath, destPath, callBack, skipIfExists = False):
    try:
        sftp = SFTPClient.from_transport(dest.connection)

        # This is pretty awful, if the file doesn't exists it throws
        # an exception, and then we should continue

        # Assume true
        exists = True

        if skipIfExists:
            try:
                sftp.stat(destPath)
            except FileNotFoundError as e:
                exists = False

        if not skipIfExists or not exists:
            renameUpload(sftp, srcPath, destPath, callBack)
        else:
            callBack()

        return True
    except Exception as e:
        print('*** Caught exception: %s: %s' % (e.__class__, e))
        traceback.print_exc()
        sys.exit(1)
    finally:
        try:
            t.close()
        except:
            pass
    return False
Example #5
0
 def _authenticate(self):
     self._transport = SFTPTransport((self.config['sftp_host'],
                                      self.config['sftp_port']))
     self._transport.connect(username=self.config['sftp_user'],
                             password=self.config['sftp_password'])
     self.session = SFTPClient.from_transport(self._transport)
     logging.info('SFTP Authorization succeed')
Example #6
0
 def scp(self, srcFile, destPath):
     transport = Transport((self.host, int(self.port)))
     transport.connect(username=self.user, password=self.passwd)
     sftp = SFTPClient.from_transport(transport)
     try:
         sftp.put(srcFile, destPath)
     except IOError as e:
         raise e
Example #7
0
 def __init__(self, host, port, username, password):
     self.host = host
     self.port = port
     self.username = username
     self.password = password
     self.t = Transport((self.host, self.port))
     self.t.connect(username=self.username, password=self.password)   # 注意:此处需要用默认传参方式传参
     self.sftp = SFTPClient.from_transport(self.t)
Example #8
0
 def copy(self, local_file, remote_file):  
   if self.localMachine:
     command = "cp %s %s" % (local_file, remote_file)
     return self.execute_command(command)
   else:  
     if not hasattr(self, 'ssh_connection'):
       self.__get_ssh_connection()
     sftp = SFTPClient.from_transport(self.ssh_connection.get_transport())
     obj = sftp.put(local_file, remote_file)
     return (0, "Copied to %s" % obj.__str__(), "")
Example #9
0
 def sftpOpenConnection(self, target):
     """ opens an sftp connection to the given target host;
         credentials are taken from /etc/ssh/sftp_passwd """
     from paramiko import Transport, SFTPClient
     from paramiko.util import load_host_keys
     hostname, username, password = self.getCredentials(target)
     hostkeys = load_host_keys('/etc/ssh/ssh_known_hosts')
     hostkeytype, hostkey = hostkeys[hostname].items()[0]
     trans = Transport((hostname, 22))
     trans.connect(username=username, password=password, hostkey=hostkey)
     return SFTPClient.from_transport(trans)
Example #10
0
    def release_success(self, build_info):
        def progress(filename, transferred, total):
            print(
                colorize('fuchsia', ' * %s transfer in progress: %02d%%.\r' % (filename, transferred * 100 / total)),
                end='\r'
            )

        ssh = SSHClient()
        try:
            ssh.set_missing_host_key_policy(AutoAddPolicy())
            ssh.connect(self.ssh['server'], username=self.ssh['user'], key_filename=self.ssh['pkey'])
        except SSHException:
            raise XUtilsError('Unable to establish a SSH connection to %s' % self.ssh['server'])

        dest_dir = '/'.join([
            self.ssh['base_dir'], build_info['category'], build_info['pkg_name'], build_info['version'],
            build_info['arch']
        ])
        stdin, stdout, stderr = ssh.exec_command('mkdir -p %s' % dest_dir)  # pylint: disable=unused-variable
        if stderr.read():
            raise XUtilsError('Unable to create directory %s on server %s' % (dest_dir, self.ssh['server']))

        src_dir = str()
        for f in listdir(self.cfg['build']['workdir']):
            if f.endswith('_root.tar.gz'):
                src_dir = self.cfg['build']['workdir']
        if not src_dir:
            src_dir = self.cfg['release']['archive_dir']
        files = [
            '%s-%s_root.tar.gz' % (build_info['pkg_name'], build_info['version']),
            '%s-%s_debuginfo.tar.gz' % (build_info['pkg_name'], build_info['version']),
            '%s-%s_root.tar.gz.gpg' % (build_info['pkg_name'], build_info['version']),
        ]
        try:
            sftp = SFTPClient.from_transport(ssh.get_transport())
            sftp.chdir(dest_dir)
        except SSHException:
            raise XUtilsError('Unable to negotiate a SFTP session for %s' % self.ssh['server'])

        for f in files:
            filepath = realpath(src_dir + '/' + f)
            if not exists(filepath):
                self.info(colorize('yellow', '%s not found => skip it.' % f))
                continue
            my_callback = partial(progress, f)
            remote_attr = sftp.put(filepath, f, callback=my_callback)
            if stat(filepath).st_size == remote_attr.st_size:
                self.info('%s successfully copied on server %s.' % (f, self.ssh['server']))
            else:
                raise XUtilsError('Copy of %s on server %s is corrupted.' % (f, self.ssh['server']))

        sftp.close()
        ssh.close()
Example #11
0
    def _connect(self):
        self._conn = Transport((self._host, self._port))
        self._conn.window_size = pow(2, 27)
        self._conn.packetizer.REKEY_BYTES = pow(2, 32)
        self._conn.packetizer.REKEY_PACKETS = pow(2, 32)
        if self._auth == 'key':
            pkey = RSAKey.from_private_key_file(self._key_path)
            self._conn.connect(username=self._user, pkey=pkey)
        else:
            self._conn.connect(username=self._user, password=self._passwd)

        self._client = SFTPClient.from_transport(self._conn)
Example #12
0
 def _connect(self):
     if self._conn is not None:
         log.warn("Already connected to sftp server")
         return
     try:
         self._conn = Transport((self._host, self._port))
         self._conn.window_size = pow(2, 27)
         self._conn.packetizer.REKEY_BYTES = pow(2, 32)
         self._conn.packetizer.REKEY_PACKETS = pow(2, 32)
         self._conn.connect(username=self._user, password=self._passwd)
         self._client = SFTPClient.from_transport(self._conn)
     except Exception as e:
         raise IrmaSFTPError("{0}".format(e))
Example #13
0
 def sftpOpen(self, target):
     """ opens an sftp connection to the given target host;
         credentials are taken from /etc/ssh/sftp_passwd """
     from paramiko import Transport, SFTPClient
     #from paramiko.util import load_host_keys
     print '>>> sftpOpen self.username: '******'             self.password: '******'             self.hostkey: ', self.password
     print '             self.hostname: ', self.hostname
     trans = Transport((self.hostname, 22))
     trans.banner_timeout = 120
     trans.connect(username=self.username,
                   password=self.password,
                   hostkey=self.hostkey)
     return SFTPClient.from_transport(trans)
Example #14
0
 def _putFile(self, data, connection):
     """ Puts the given file on the TSM host. """
     
     sftp = SFTPClient.from_transport(connection)
     try:
         try:
             remoteFileObject = sftp.open(self._persistenceIdentifier, "w")
             block = data.read(_BLOCK_SIZE)
             while len(block) > 0:
                 remoteFileObject.write(block)
                 block = data.read(_BLOCK_SIZE)
         except (IOError, SSHException), sshException:
             errorMessage = "Cannot transfer data to TSM host!\nReason: '%s'" % str(sshException)
             raise PersistenceError(errorMessage)
     finally:
         data.close()
         sftp.close()
Example #15
0
def listdir(hostname, path="/var/tmp", filter="", port=1035, username="", password=""):
    """
        paramiko sftp listdir wrapper, with option to filter files
    """
    # Paramiko client configuration

    t = Transport((hostname, port))
    t.connect(username=username, password=password)
    sftp = SFTPClient.from_transport(t)

    try:
        rex = re.compile(filter)
    except:
        print "Invalid regular expression: " + filter
        sys.exit(1)

    return [x for x in sftp.listdir(path) if rex.match(x)]
Example #16
0
    def upload_report_to_sftp(self, client_id, report_date, absolute_filename):
        """
        Upload the given file, using SFTP, to the configured FTP server. The
        file should be uploaded to the appropriate directory for the specified
        client and the date of the report.
        """
        try:
            client = Clients.objects.get(id=client_id)
        except Clients.DoesNotExist:
            logger.exception(u'No configuration for client {0}.'.format(client_id))
            raise

        filename = basename(absolute_filename)
        base_folder, env_folder, year_folder, month_folder = self._get_sftp_dirs(client, report_date)

        try:
            logger.debug(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME))
            transport = Transport((settings.SFTP_SERVER, settings.SFTP_PORT))
            transport.connect(username=settings.SFTP_USERNAME, password=settings.SFTP_PASSWORD)
            sftp = SFTPClient.from_transport(transport)

            logger.debug(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder))
            sftp.chdir(base_folder)
            self._make_or_change_sftp_dir(sftp, env_folder)
            self._make_or_change_sftp_dir(sftp, year_folder)
            self._make_or_change_sftp_dir(sftp, month_folder)

            logger.debug(u'SFTP uploading {0}'.format(filename))
            sftp.put(absolute_filename, filename)
        except Exception:
            logger.exception(u'Unrecoverable exception during SFTP upload process.')
            raise
        finally:
            logger.debug(u'SFTP logging off')

            try:
                sftp.close()
            except Exception:
                logger.exception(u'SFTP exception while closing SFTP session.')

            try:
                transport.close()
            except Exception:
                logger.exception(u'SFTP exception while closing SSH connection.')
Example #17
0
def sftp_connect(config):
    try:
        hostname = config["hostname"]
        port = int(config["port"])
        username = config["username"]
        pkeyfile = config["privatekey_file"]
        pkeypassword = config["privatekey_passphrase"]
        host_keys = load_host_keys(config["hostkeys_file"])
        hostkeytype, hostkey = get_hostkeytype_and_hostkey(host_keys,
                                                           config["hostname"])
        t = Transport((hostname, port))
        pkey = get_privatekey_from_file(pkeyfile, pkeypassword) 
        t.connect(username=username, pkey=pkey, hostkey=hostkey)

        return SFTPClient.from_transport(t)

    except SSHException, e:
        print " [*] " + str(e)
        sys.exit(1)
Example #18
0
    def _getFile(self, connection):
        """ Transfers the given file from the TSM host to the local file system. """

        try:
            temporaryFileObject = tempfile.TemporaryFile()
            sftp = SFTPClient.from_transport(connection)
            try:
                temporaryFileObject.seek(0)
                remoteFileObject = sftp.open(self._persistenceIdentifier)
                block = remoteFileObject.read(_BLOCK_SIZE)
                while len(block) > 0:
                    temporaryFileObject.write(block)
                    block = remoteFileObject.read(_BLOCK_SIZE)
                sftp.remove(self._persistenceIdentifier)
                temporaryFileObject.seek(0)
                return temporaryFileObject
            except (IOError, SSHException), error:
                errorMessage = "Cannot retrieve file from TSM host!\nReason: '%s'" % str(error)
                raise PersistenceError(errorMessage)
        finally:
            sftp.close()
Example #19
0
 def connect(self):
   """ Get a handle to a remote connection """
   # Check URL
   schema = urlparse(self.url)
   if schema.scheme == 'sftp':
     self.transport = Transport((schema.hostname, int(schema.port)))
   else:
     raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme))
   # Add authentication to transport
   try:
     if self.password:
       self.transport.connect(username=self.user_name, password=self.password)
     elif self.private_key:
       self.transport.connect(username=self.user_name,
                              pkey=RSAKey.from_private_key(StringIO(self.private_key)))
     else:
       raise SFTPError("No password or private_key defined")
     # Connect
     self.conn = SFTPClient.from_transport(self.transport)
   except (socket.gaierror,error), msg:
     raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
Example #20
0
 def connect(self):
   """ Get a handle to a remote connection """
   # Check URL
   schema = urlparse(self.url)
   if schema.scheme == 'sftp':
     hostname = schema.hostname
     port = int(schema.port)
     # Socket creation code inspired from paramiko.Transport.__init__
     # with added bind support.
     for family, socktype, _, _, _ in getaddrinfo(
           hostname, port, AF_UNSPEC, SOCK_STREAM,
         ):
       if socktype == SOCK_STREAM:
         sock = socket(family, SOCK_STREAM)
         if self.bind_address:
           # XXX: Expects bind address to be of same family as hostname.
           # May not be easy if name resolution is involved.
           # Try to reconciliate them ?
           sock.bind((self.bind_address, 0))
         retry_on_signal(lambda: sock.connect((hostname, port)))
         break
     else:
       raise SFTPError('No suitable socket family found')
     self.transport = Transport(sock)
   else:
     raise SFTPError('Not a valid sftp url %s, type is %s' %(self.url, schema.scheme))
   # Add authentication to transport
   try:
     if self.password:
       self.transport.connect(username=self.user_name, password=self.password)
     elif self.private_key:
       self.transport.connect(username=self.user_name,
                              pkey=RSAKey.from_private_key(StringIO(self.private_key)))
     else:
       raise SFTPError("No password or private_key defined")
     # Connect
     self.conn = SFTPClient.from_transport(self.transport)
   except (gaierror, error), msg:
     raise SFTPError(str(msg) + ' while establishing connection to %s' % (self.url,))
Example #21
0
def get_sftp_conn(config):
    """Make a SFTP connection, returns sftp client and connection objects"""
    remote = config.get('remote_location')
    parts = urlparse(remote)

    if ':' in parts.netloc:
        hostname, port = parts.netloc.split(':')
    else:
        hostname = parts.netloc
        port = 22
    port = int(port)

    username = config.get('remote_username') or getuser()
    luser = get_local_user(username)
    sshdir = get_ssh_dir(config, luser)
    hostkey = get_host_keys(hostname, sshdir)

    try:
        sftp = None
        keys = get_ssh_keys(sshdir)
        transport = Transport((hostname, port))
        while not keys.empty():
            try:
                key = PKey.from_private_key_file(keys.get())
                transport.connect(
                    hostkey=hostkey,
                    username=username,
                    password=None,
                    pkey=key)
                sftp = SFTPClient.from_transport(transport)
                break
            except (PasswordRequiredException, SSHException):
                pass
        if sftp is None:
            raise SaChannelUpdateTransportError("SFTP connection failed")
        return sftp, transport
    except BaseException as msg:
        raise SaChannelUpdateTransportError(msg)
Example #22
0
    def __call__(self):
        registry = getUtility(IRegistry)
        recensio_settings = registry.forInterface(IRecensioSettings)
        host = recensio_settings.xml_export_server
        username = recensio_settings.xml_export_username
        password = recensio_settings.xml_export_password
        if not host:
            return 'no host configured'
        log.info("Starting XML export to sftp")

        exporter = getUtility(IFactory, name='chronicon_exporter')()
        export_xml = exporter.get_export_obj(self.context)
        if export_xml is None:
            msg = "Could not get export file object: {0}".format(
                exporter.export_filename)
            log.error(msg)
            return msg

        zipstream = export_xml.getFile()
        try:
            transport = Transport((host, 22))
            transport.connect(username=username, password=password)
            sftp = SFTPClient.from_transport(transport)
            attribs = sftp.putfo(zipstream.getBlob().open(), self.filename)
        except (IOError, SSHException) as ioe:
            msg = "Export failed, {0}: {1}".format(ioe.__class__.__name__, ioe)
            log.error(msg)
            return msg
        if attribs.st_size == zipstream.get_size():
            msg = "Export successful"
            log.info(msg)
            return msg
        else:
            msg = "Export failed, {0}/{1} bytes transferred".format(
                attribs.st_size, zipstream.get_size())
            log.error(msg)
            return msg
Example #23
0
# -*- coding: utf-8 -*- 
__author__ = 'Administrator'


from paramiko import Transport,RSAKey,SFTPClient
import os

host="54.187.91.18"
port=22

private_key_file=os.path.expanduser("~/.ssh/id_rsa")
key=RSAKey.from_private_key_file(private_key_file)


t=Transport((host,port))
t.connect(username="******",pkey=key)

sftp=SFTPClient.from_transport(t)
try:
	sftp.put("/data/websites/test/a.php","/data/websites/")
except Exception,e:
	print(e)

sftp.close()
t.close()

Example #24
0
    def Link(self,\
            sftp=None,\
            ssh=True):
        self.sock = socket()
        self.sock.settimeout(self.timeout)
        self.Errors = []
        self.protocol = []
        self.logger.info('Trying to start link using parameters [SSH=%s,SFTP=%s]'%(ssh,sftp))
        try:
            self.sock.connect((self.remoteMachine, self.port))

        except:
            self.Unlink()
            self.logger.error('Link: Could not open a socket [remoteMachine = %s, port = %s]' % (self.remoteMachine, self.port))
            

        self.transport = Transport(self.sock)

        try:
            if not self.password:
                self.transport.connect(username = self.username, pkey = self.pk)
                self.logger.info('Trying to connect using private key..')
            else:
                self.transport.connect(username = self.username, password = self.password)
                self.logger.info('Trying to connect using password..')

        except:
            if not self.password:
                self.logger.error('Private Key not installed and no password specified!')
                
            self.Unlink()
            self.logger.error('Could not connect using SFTP [remoteMachine = %s, Username = %s, port = %s]' %(self.remoteMachine, self.username, self.port))
            
            sftp=None

            
        if ssh:
            try:
                self.protocol.append(mySSHClient())
                self.ssh = self.protocol[-1]
                self.ssh.set_missing_host_key_policy(AutoAddPolicy())
                self.ssh.connect(self.remoteMachine, port = self.port, username = self.username, password = self.password)
                self.logger.info ('ssh connection created..')
            except:
                self.logger.error('Could not connect using SSH [remoteMachine = %s, Username = %s, port = %s]'%(self.remoteMachine, self.username, self.port))

        if sftp:
            try:
                if ssh == None:
                    self.logger.error ('SSH must be enabled in order to connect with SFTP')
                    
                
                self.protocol.append(SFTPClient.from_transport(self.transport))
                self.sftp = self.protocol[-1]
                self.client = self.sftp
                self.logger.info('sftp connection created..')
            except:
                self.logger.error ('Could not bring up SFTP connection')
                
                self.sftp=None
    
        return ssh, sftp
Example #25
0
def upload_sftp(server_info, upload_files, delete_files, force_reupload=False):
	server, info = server_info
	t = Transport((info['host'], info['port']))
	try:
		t.connect(username=info['username'], password=info['password'])
	except AuthenticationException as e:
		print('\t\t%s' % e)
		return False, [], []


	plugins_load = []
	plugins_reload = []

	sftp = SFTPClient.from_transport(t)
	try:
		sftp.chdir(info['path'])
	except FileNotFoundError as e:
		print('\t\tCannot chdir to remote path %s: %s' % (info['path'], e))
		return False, [], []

	if delete_files:
		print('\t\tProcessing %d deletions...' % (len(delete_files)))
		for i, d in enumerate(delete_files):
			print('\t\t\t%d. %s' % (i+1, d))
			try:
				file_info = sftp.stat(info['path'] + '/' + d)
				if stat.S_ISDIR(file_info.st_mode):
					sftp.rmdir(info['path'] + '/' + d)
				else:
					sftp.remove(info['path'] + '/' + d)
			except FileNotFoundError:
				pass
			except Exception as e:
				print('\t\t\t\tFailed: %s' % e)

	for (path, file) in upload_files:
		abs_path = info['path'] + '/' + path
		relative_filepath = path + '/' + file
		print('\t\tProcessing %s' % file)

		try:
			sftp.stat(abs_path)
		except FileNotFoundError:
			print('\t\t\t%s does not exist, creating...' % path)
			
			i=1
			while i != -1:
				i = path.find('/', i)
				abs_path = info['path'] + path[0:i]
				if i != -1:
					try:
						sftp.stat(abs_path)
						i += 1
					except FileNotFoundError:	
						sftp.mkdir(abs_path)
					
			sftp.mkdir(info['path'] + path)

		local_filepath = 'upload' + relative_filepath

		try:
			fileinfo = sftp.stat(info['path'] + relative_filepath)
			print('\t\t\tRemote file found')

			upload = False
			if (force_reupload):
				print('\t\t\tForcing reupload')
				upload = True
			else:
				lsize = os.path.getsize(local_filepath)
				rsize = fileinfo.st_size

				if lsize != rsize:
					print('\t\t\tSize mismatch -- Reuploading')
					upload = True

			if upload:
				sftp.put(local_filepath, info['path'] + relative_filepath)
				
				if (file[-4:] == '.smx'):
					plugins_reload.append(file[:-4])

		except FileNotFoundError:
			print('\t\t\tUploading')

			sftp.put(local_filepath, info['path'] + relative_filepath)

			if (file[-4:] == '.smx'):
				plugins_load.append(file[:-4])
	
	t.close()
	print('\t\tLogout')

	return True, plugins_load, plugins_reload
Example #26
0
async def pull_file(ftp: paramiko.SFTPClient, ftp_path: str, local_path: str):
    """Pulls file from remote path to local path only if out of date."""
    if compare_files(ftp, ftp_path, local_path):
        ftp.get(ftp_path, local_path)
    else:
        print("Local is up to date.")
Example #27
0
    def submitJobToFramework(self, **kwargs):
        jobCommand = 'job'
        
        daemonArgs = DaemonArgs(self.config)
        daemonArgs.command = jobCommand
        unScheduledJob = kwargs['unScheduledJob']
        
        is_fileFeeder = False
        fileFeederUploadedFile = None
        del daemonArgs.param[:]

        # go through all parameters
        for parameter in unScheduledJob.parameters.all():

            # add parameter to daemonArgs.param
            if parameter.service and parameter.param_key and parameter.param_value:

                # check if a file feeder is used
                if parameter.service == settings.FILE_FEEDER_ID:
                    is_fileFeeder = True
                    fileFeederUploadedFile = parameter.param_value

                    remoteFeederFile = os.path.join(self.sftpRemotePath, parameter.param_value)
                    parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, remoteFeederFile )
                else:
                    parameterString = '%s.%s=%s' % ( parameter.service, parameter.param_key, parameter.param_value )

                self.logger.debug("add parameter string: %s" % parameterString)
                daemonArgs.param.append([parameterString])

        # in case of a filefeeder upload file to framework server
        if is_fileFeeder:
            self.logger.debug("is file feeder")
            sftp = None
            transport = None
            try:
                transport = Transport((self.sftpHost, self.sftpPort))
                if self.sftpPassword:
                    transport.connect(username=self.sftpUsername, password=self.sftpPassword)
                else:
                    privateKey = None
                    if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'rsa':
                        privateKey = RSAKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword )
                    if self.sftpPrivateKeyType and self.sftpPrivateKeyType.lower() == 'dss':
                        privateKey = DSSKey.from_private_key_file(self.sftpPrivateKey, password=self.sftpPrivateKeyPassword )

                    transport.connect(username=self.sftpUsername, pkey=privateKey)

                sftp = SFTPClient.from_transport(transport)

                filePath = os.path.join( settings.MEDIA_ROOT, fileFeederUploadedFile )
                remotePath = os.path.join( self.sftpRemotePath, fileFeederUploadedFile )

                self.logger.debug("uploading file from %s to %s on remote machine" % (filePath, remotePath))

                sftp.put(filePath, remotePath)
#                            sftp.put(filePath, remotePath, confirm=False)
                sftp.chmod( remotePath, 0644 )

                self.logger.debug("put OK")

            except IOError as e:
                self.logger.error("IOError: %s. Will continue with next scheduled job." % e)
                self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
            except PasswordRequiredException as e:
                self.logger.error("PasswordRequiredException: %s. Will continue with next scheduled job." % e)
                self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
            except SSHException as e:
                self.logger.error("SSH Exception: %s. Will continue with next scheduled job." % e)
                self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
            except Exception as e:
                self.logger.error("Unkown SFTP problem. Will continue with next scheduled job. %s" % e)
                self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
            finally:
                if sftp is not None:
                    sftp.close()
                if transport is not None:
                    transport.close()
                
        # set job workflow
        daemonArgs.jd_workflow = unScheduledJob.workflow.name

        frameworkJobId = None
        
        try:
            setattr(daemonArgs, jobCommand, 'submit')
            frameworkJobId = self.sendFrameworkCommand(jobCommand, daemonArgs)
            self.saveJob(Job.PROCESSING_STATUS, frameworkJobId, unScheduledJob)
        except WorkflowNotDeployedException:
            # The workflow is not deployed in the framework. To prevent the scheduler retrying continuously
            # we disable this job
            unScheduledJob.status = Schedule.DEACTIVATE_STATUS
            unScheduledJob.save()
        except:
            self.saveJob(Job.FAILED_STATUS, None, unScheduledJob)
        finally:
            daemonArgs.clean(jobCommand)
        
        if unScheduledJob.scheduled_start is not None:
            unScheduledJob.status = Schedule.DEACTIVATED_STATUS
            unScheduledJob.save()
Example #28
0
def __upload(
    sftp_client: paramiko.SFTPClient, filename: str, local_path: str, remote_path: str
):
    with open(os.path.join(local_path, filename), "rb") as f:
        data = f.read()
    sftp_client.open(os.path.join(remote_path, filename), "wb").write(data)
    def connect(self, host, port=22, user=None, passw=None, cert=None, path='/', timeout=10):
        """Method connects to server

        Args:
           host (str): server host
           port (int): server port, default protocol port
           user (str): username
           passw (str): password
           cert (str): path to certificate file
           path (str): server path
           timeout (int): timeout

        Returns:
           bool: result

        Raises:
           event: ftp_before_connect
           event: ftp_after_connect

        """

        try:

            message = '{0}/{1}@{2}:{3}{4} cert:{5}, timeout:{6}'.format(
                user, passw, host, port, path, cert, timeout)
            self._mh.demsg('htk_on_debug_info', self._mh._trn.msg(
                'htk_ftp_connecting', message), self._mh.fromhere())

            ev = event.Event(
                'ftp_before_connect', host, port, user, passw, cert, path, timeout)
            if (self._mh.fire_event(ev) > 0):
                host = ev.argv(0)
                port = ev.argv(1)
                user = ev.argv(2)
                passw = ev.argv(3)
                cert = ev.argv(4)
                path = ev.argv(5)
                timeout = ev.argv(6)

            self._host = host
            self._port = port
            self._user = user
            self._passw = passw
            self._cert = cert

            if (ev.will_run_default()):
                setdefaulttimeout(timeout)
                t = Transport((host, self._port))

                if (user != None or cert != None):
                    pkey = RSAKey.from_private_key_file(
                        self._cert) if (cert != None) else None
                    t.connect(username=user, password=passw, pkey=pkey)
                    self._client = SFTPClient.from_transport(t)

                self._is_connected = True

                if (path != None):
                    self.change_dir(path)

            ev = event.Event('ftp_after_connect')
            self._mh.fire_event(ev)

            return True

        except (SSHException, NoValidConnectionsError, error) as ex:
            self._mh.demsg(
                'htk_on_error', 'error: {0}'.format(ex), self._mh.fromhere())
            return False
Example #30
0
 def __connect(self):
     self.t = Transport((self.config['hostname'], self.config['port']))
     self.t.connect(username = self.config['user'], pkey = self.auth_key)
     self.client = SFTPClient.from_transport(self.t)
     self.client.chdir(self.path)
Example #31
0
 def sftp_put_file(self, local_path, dest_path):
     self.t = Transport(sock=(self.host, self.ssh_port))
     self.keyfile = self.__get_ssh_key()
     self.t.connect(username=self.user, pkey=self.keyfile)
     self.sftp = SFTPClient.from_transport(self.t)
     self.sftp.put(local_path, dest_path)
Example #32
0
	def _process_directory(
		self,
		level: int,
		sftp: paramiko.SFTPClient,
		remote_root: Path,
		remote_path: Path,
		local_targetdir: Path,
		entry: BackupEntry
	):
		isroot = False
		options = entry.get_options()
		has_options = not is_empty_dict(options)
		path_rootindex = None
		recurse = False

		if has_options:
			if "recurse" in options and not self._check_option_ignored("recures"):
				recurse = options["recurse"]
			if "path_rootindex" in options and not self._check_option_ignored("path_rootindex"):
				path_rootindex = options["path_rootindex"]

		if len(remote_path.parents) == 0:
			isroot = True

		if isroot:
			localdir = local_targetdir
		else:
			if path_rootindex is not None and is_integer(path_rootindex):
				f = path_from_partindex(remote_path, path_rootindex)
				localdir = local_targetdir.joinpath(f)
			else:
				localdir = local_targetdir.joinpath(str(remote_path)[1:])

		remote_filenode = None
		tabs2 = repeat("\t", level + 1)

		do_transfer = self._check_folder_with_options(remote_root, remote_path, options)

		if do_transfer is not None:
			self.info("{}Excluding '{}' due to json-folder-option {}".format(
				tabs2,
				remote_path,
				do_transfer
			))
			return

		if not localdir.exists():
			self.info("{}Creating parent folders '{}'".format(tabs2, localdir))
			makedirs(str(localdir))

		try:
			filelist = sftp.listdir(str(remote_path))

			if len(filelist) > 0:
				for filenode in filelist:
					remote_filenode = remote_path.joinpath(filenode)
					remote_stat = sftp.lstat(str(remote_filenode))

					if stat.S_ISDIR(remote_stat.st_mode):
						if recurse is True:
							self.info("\n{}Recursing into sub-directory '{}'".format(
								tabs2, remote_filenode
							))
							self._process_directory(
								level + 1, sftp,
								remote_path,
								remote_filenode,
								local_targetdir,
								entry
							)
					else:
						localfile = localdir.joinpath(filenode)
						self._download_file(level, sftp, remote_root, localfile, remote_filenode, entry)

		except Exception as e:
			if isinstance(e, PermissionError):
				self.info("{}PermissionError while listing contents of {}\n".format(
					tabs2,
					remote_filenode if remote_filenode is not None else remote_path
				))
			elif isinstance(e, JobException):
				raise e
			else:
				raise JobException(e, 1)
Example #33
0
 def connect(self):
     self._transport = Transport((self._host, self._port))
     self._transport.connect(username=self._user, password=self._pass)
     self._client = SFTPClient.from_transport(self._transport)
Example #34
0
def _get_sftp_connection(host):
    transport = host.connection.get_transport()
    return SFTPClient.from_transport(transport)
Example #35
0
 def copy(self, local_file, remote_file):    
       if not hasattr(self, 'ssh_connection'):
           self.__get_ssh_connection()
       sftp = SFTPClient.from_transport(self.ssh_connection.get_transport())
       sftp.put(local_file, remote_file)
Example #36
0
from traceback import format_exc
import sys
#import FEWSConnect
import logging
from paramiko import Transport
from paramiko import SFTPClient

if (__name__ == '__main__'):
    host = "ftp.ncep.noaa.gov"
    port = 22  #after much testing....port 22 is the place to be
    #usr = "******"
    #pw = "Shenandoah22"
    path = "/pub/data/nccf/com/nwm/prod/"  #r"/Distribution/RHA-Data/rhapub5/hefs_icprb"
    lookback = 5

    #ftp://ftp.ncep.noaa.gov/pub/data/nccf/com/nwm/prod

    #https://ftp.ncep.noaa.gov/data/nccf/com/nwm/prod/

    # try:
    #     transport = Transport((host, port))
    #     transport.connect()#username = usr, password = pw)
    #     sftp = SFTPClient.from_transport(transport)
    #     print("1 it's alive...sftp object up and running")
    # except:
    #     print("1 failed to connect")

    transport = Transport((host, port))
    transport.connect()  #username = usr, password = pw)
    sftp = SFTPClient.from_transport(transport)
Example #37
0
    def create_connection(cls, host, port, username, password):

        transport = Transport(sock=(host, port))
        transport.connect(username=username, password=password)
        cls._connection = SFTPClient.from_transport(transport)
 def __init__(self):
     self.transport = Transport(sock=(self.HOST, self.PORT))
     self.transport.connect(username=self.USERNAME, password=self.PASSWORD)
     self.ftp_connection = SFTPClient.from_transport(self.transport)
Example #39
0
    def upload_report(self, filepath, file_name, report_date):
        """
        >>> a = CSVgenerator('123')
        >>> a.upload_report(None, 'upload.txt', '2014-05-06')
        """
        # appropriate changes required for doc test to run - dev only.
        # test_file_path = join(settings.REPORTS_ROOT, '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv')
        # # trace.info('----- {}'.format(abspath(test_file_path)))
        # absolute_filename = abspath(test_file_path)
        # test_filename = '16b6a354-ff32-4be4-b648-fe51fc5b1508.csv'

        #TODO remove after QA testing.
        if not filepath:
            test_file_path = join(settings.REPORTS_ROOT, file_name)
            absolute_filename = abspath(test_file_path)
        else:
            absolute_filename = abspath(filepath)

        try:
            report_date = report_date if isinstance(report_date, str) else report_date.strftime(settings.DATE_FORMAT_YMD)
        except Exception:
            log.exception('')

        try:
            #TODO use sftp credentials, once available.
            #TODO get client upload location for per-client report.
            # ftp_client_dir = client.ftp_client_dir
            # if ftp_client_dir == '':
            #     ftp_client_dir = getattr(settings, 'DEFAULT_SFTP_LOCATION', 'default')
            #     logger.exception(u'No FTP configuration for client {} using default value.'.format(client.name))
            #

            year_folder, month_folder, _ = report_date.split('-')

            base_folder_path = self._sftp_settings.get('path', '')

            base_path = base_folder_path.split('/')
            base_folders = [i for i in base_path[1:-1]]
            base_folder = '/' + join(*base_folders)
            env_folder = str(base_path[-1:][0])

            from paramiko import Transport, SFTPClient
            try:
                log.info(u'SFTP logging on to {0} as {1}'.format(settings.SFTP_SERVER, settings.SFTP_USERNAME))
                transport = Transport((
                    self._sftp_settings.get('server', ''),
                    self._sftp_settings.get('port', 22))
                )
                transport.connect(
                    username=self._sftp_settings.get('username', ''),
                    password=self._sftp_settings.get('password', '')
                )
                sftp = SFTPClient.from_transport(transport)
                log.info(u'SFTP dir {0}/{1}/{2}/{3}'.format(base_folder, env_folder, year_folder, month_folder))
                try:
                    sftp.chdir(base_folder)
                except Exception:
                    log.debug('Unable to change to base folder on ftp server.')

                self._make_or_change_sftp_dir(sftp, env_folder)
                self._make_or_change_sftp_dir(sftp, year_folder)
                self._make_or_change_sftp_dir(sftp, month_folder)

                log.debug(u'SFTP uploading {0}'.format(filepath))
                sftp.put(absolute_filename, file_name)
            except Exception:
                log.exception(u'Unrecoverable exception during SFTP upload process.')

            finally:
                log.debug(u'SFTP logging off')

                try:
                    sftp.close()
                except Exception:
                    log.exception(u'SFTP exception while closing SFTP session.')

                try:
                    transport.close()
                except Exception:
                    log.exception(u'SFTP exception while closing SSH connection.')

        except Exception:
            log.debug('Error while uploading report.')