Beispiel #1
0
    def get_url(self,
                url,
                dest,
                makedirs=False,
                saltenv='base',
                env=None,
                no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path \'{0}\' is not absolute'.format(url_data.path))
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:

                def s3_opt(key, default=None):
                    '''Get value of s3.<key> from Minion config or from Pillar'''
                    if 's3.' + key in self.opts:
                        return self.opts['s3.' + key]
                    try:
                        return self.opts['pillar']['s3'][key]
                    except (KeyError, TypeError):
                        return default

                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=s3_opt('key'),
                                    keyid=s3_opt('keyid'),
                                    service_url=s3_opt('service_url'),
                                    verify_ssl=s3_opt('verify_ssl', True),
                                    location=s3_opt('location'))
                return dest
            except Exception as exc:
                raise MinionError(
                    'Could not fetch from {0}. Exception: {1}'.format(
                        url, exc))
        if url_data.scheme == 'ftp':
            try:
                ftp = ftplib.FTP(url_data.hostname)
                ftp.login()
                with salt.utils.fopen(dest, 'wb') as fp_:
                    ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
                return dest
            except Exception as exc:
                raise MinionError(
                    'Could not retrieve {0} from FTP server. Exception: {1}'.
                    format(url, exc))

        if url_data.scheme == 'swift':
            try:

                def swift_opt(key, default):
                    '''Get value of <key> from Minion config or from Pillar'''
                    if key in self.opts:
                        return self.opts[key]
                    try:
                        return self.opts['pillar'][key]
                    except (KeyError, TypeError):
                        return default

                swift_conn = SaltSwift(swift_opt('keystone.user', None),
                                       swift_opt('keystone.tenant', None),
                                       swift_opt('keystone.auth_url', None),
                                       swift_opt('keystone.password', None))

                swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            netloc = url_data.netloc
            at_sign_pos = netloc.rfind('@')
            if at_sign_pos != -1:
                netloc = netloc[at_sign_pos + 1:]
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path, url_data.params,
                 url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            query = salt.utils.http.query(fixed_url,
                                          text=True,
                                          username=url_data.username,
                                          password=url_data.password,
                                          **get_kwargs)
            if 'text' not in query:
                raise MinionError('Error: {0}'.format(query['error']))
            if no_cache:
                return query['body']
            else:
                dest_tmp = "{0}.part".format(dest)
                with salt.utils.fopen(dest_tmp, 'wb') as destfp:
                    destfp.write(query['body'])
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code, url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #2
0
    def get_url(self,
                url,
                dest,
                makedirs=False,
                saltenv='base',
                env=None,
                no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Carbon',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Carbon.')
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path \'{0}\' is not absolute'.format(url_data.path))
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:

                def s3_opt(key, default=None):
                    '''Get value of s3.<key> from Minion config or from Pillar'''
                    if 's3.' + key in self.opts:
                        return self.opts['s3.' + key]
                    try:
                        return self.opts['pillar']['s3'][key]
                    except (KeyError, TypeError):
                        return default

                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=s3_opt('key'),
                                    keyid=s3_opt('keyid'),
                                    service_url=s3_opt('service_url'),
                                    verify_ssl=s3_opt('verify_ssl', True),
                                    location=s3_opt('location'))
                return dest
            except Exception as exc:
                raise MinionError(
                    'Could not fetch from {0}. Exception: {1}'.format(
                        url, exc))
        if url_data.scheme == 'ftp':
            try:
                ftp = ftplib.FTP(url_data.hostname)
                ftp.login()
                with salt.utils.fopen(dest, 'wb') as fp_:
                    ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
                return dest
            except Exception as exc:
                raise MinionError(
                    'Could not retrieve {0} from FTP server. Exception: {1}'.
                    format(url, exc))

        if url_data.scheme == 'swift':
            try:

                def swift_opt(key, default):
                    '''Get value of <key> from Minion config or from Pillar'''
                    if key in self.opts:
                        return self.opts[key]
                    try:
                        return self.opts['pillar'][key]
                    except (KeyError, TypeError):
                        return default

                swift_conn = SaltSwift(swift_opt('keystone.user', None),
                                       swift_opt('keystone.tenant', None),
                                       swift_opt('keystone.auth_url', None),
                                       swift_opt('keystone.password', None))

                swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            netloc = url_data.netloc
            at_sign_pos = netloc.rfind('@')
            if at_sign_pos != -1:
                netloc = netloc[at_sign_pos + 1:]
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path, url_data.params,
                 url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            # Tornado calls streaming_callback on redirect response bodies.
            # But we need streaming to support fetching large files (> RAM avail).
            # Here we working this around by disabling recording the body for redirections.
            # The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
            # when we'll deprecate Tornado<4.3.0.
            # See #27093 and #30431 for details.

            # Use list here to make it writable inside the on_header callback. Simple bool doesn't
            # work here: on_header creates a new local variable instead. This could be avoided in
            # Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
            write_body = [False]

            def on_header(hdr):
                try:
                    hdr = parse_response_start_line(hdr)
                except HTTPInputError:
                    # Not the first line, do nothing
                    return
                write_body[0] = hdr.code not in [301, 302, 303, 307]

            if no_cache:
                result = []

                def on_chunk(chunk):
                    if write_body[0]:
                        result.append(chunk)
            else:
                dest_tmp = "{0}.part".format(dest)
                destfp = salt.utils.fopen(dest_tmp, 'wb')

                def on_chunk(chunk):
                    if write_body[0]:
                        destfp.write(chunk)

            query = salt.utils.http.query(fixed_url,
                                          stream=True,
                                          streaming_callback=on_chunk,
                                          header_callback=on_header,
                                          username=url_data.username,
                                          password=url_data.password,
                                          **get_kwargs)
            if 'handle' not in query:
                raise MinionError('Error: {0}'.format(query['error']))
            if no_cache:
                return ''.join(result)
            else:
                destfp.close()
                destfp = None
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code, url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #3
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path {0!r} is not absolute'.format(url_data.path))
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            if salt.utils.is_windows():
                netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
            else:
                netloc = url_data.netloc
            dest = salt.utils.path_join(self.opts['cachedir'], 'extrn_files',
                                        saltenv, netloc, url_data.path)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(
                    method='GET',
                    bucket=url_data.netloc,
                    path=url_data.path[1:],
                    return_bin=False,
                    local_file=dest,
                    action=None,
                    key=self.opts.get('s3.key', None),
                    keyid=self.opts.get('s3.keyid', None),
                    service_url=self.opts.get('s3.service_url', None),
                    verify_ssl=self.opts.get('s3.verify_ssl', True))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(
                    self.opts.get('keystone.user', None),
                    self.opts.get('keystone.tenant', None),
                    self.opts.get('keystone.auth_url', None),
                    self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path, url_data.params,
                 url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url
        try:
            if requests.__version__[0] == '0':
                # 'stream' was called 'prefetch' before 1.0, with flipped meaning
                get_kwargs['prefetch'] = False
            else:
                get_kwargs['stream'] = True
            response = requests.get(fixed_url, **get_kwargs)
            with salt.utils.fopen(dest, 'wb') as destfp:
                for chunk in response.iter_content(chunk_size=32 * 1024):
                    destfp.write(chunk)
            return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code, url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
Beispiel #4
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)
        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            if salt.utils.is_windows():
                netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
            else:
                netloc = url_data.netloc
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True))
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                             self.opts.get('keystone.tenant', None),
                                             self.opts.get('keystone.auth_url', None),
                                             self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            passwd_mgr = url_passwd_mgr()
            passwd_mgr.add_password(
                None, fixed_url, url_data.username, url_data.password)
            auth_handler = url_auth_handler(passwd_mgr)
            opener = url_build_opener(auth_handler)
            url_install_opener(opener)
        else:
            fixed_url = url
        try:
            req = requests.get(fixed_url)
            with salt.utils.fopen(dest, 'wb') as destfp:
                destfp.write(req.content)
            return dest
        except HTTPError as ex:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                ex.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
        except URLError as ex:
            raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Beispiel #5
0
    def get_url(self, url, dest, makedirs=False, saltenv='base',
                no_cache=False, cachedir=None):
        '''
        Get a single file from a URL.
        '''
        url_data = urlparse(url)
        url_scheme = url_data.scheme
        url_path = os.path.join(
                url_data.netloc, url_data.path).rstrip(os.sep)

        if url_scheme and url_scheme.lower() in string.ascii_lowercase:
            url_path = ':'.join((url_scheme, url_path))
            url_scheme = 'file'

        if url_scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_path):
                raise CommandExecutionError(
                    'Path \'{0}\' is not absolute'.format(url_path)
                )
            if dest is None:
                with salt.utils.fopen(url_path, 'r') as fp_:
                    data = fp_.read()
                return data
            return url_path

        if url_scheme == 'salt':
            result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
            if result and dest is None:
                with salt.utils.fopen(result, 'r') as fp_:
                    data = fp_.read()
                return data
            return result

        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv, cachedir=cachedir)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                def s3_opt(key, default=None):
                    '''Get value of s3.<key> from Minion config or from Pillar'''
                    if 's3.' + key in self.opts:
                        return self.opts['s3.' + key]
                    try:
                        return self.opts['pillar']['s3'][key]
                    except (KeyError, TypeError):
                        return default
                self.utils['s3.query'](method='GET',
                                       bucket=url_data.netloc,
                                       path=url_data.path[1:],
                                       return_bin=False,
                                       local_file=dest,
                                       action=None,
                                       key=s3_opt('key'),
                                       keyid=s3_opt('keyid'),
                                       service_url=s3_opt('service_url'),
                                       verify_ssl=s3_opt('verify_ssl', True),
                                       location=s3_opt('location'))
                return dest
            except Exception as exc:
                raise MinionError(
                    'Could not fetch from {0}. Exception: {1}'.format(url, exc)
                )
        if url_data.scheme == 'ftp':
            try:
                ftp = ftplib.FTP()
                ftp.connect(url_data.hostname, url_data.port)
                ftp.login(url_data.username, url_data.password)
                with salt.utils.fopen(dest, 'wb') as fp_:
                    ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
                ftp.quit()
                return dest
            except Exception as exc:
                raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))

        if url_data.scheme == 'swift':
            try:
                def swift_opt(key, default):
                    '''Get value of <key> from Minion config or from Pillar'''
                    if key in self.opts:
                        return self.opts[key]
                    try:
                        return self.opts['pillar'][key]
                    except (KeyError, TypeError):
                        return default

                swift_conn = SaltSwift(swift_opt('keystone.user', None),
                                       swift_opt('keystone.tenant', None),
                                       swift_opt('keystone.auth_url', None),
                                       swift_opt('keystone.password', None))

                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            netloc = url_data.netloc
            at_sign_pos = netloc.rfind('@')
            if at_sign_pos != -1:
                netloc = netloc[at_sign_pos + 1:]
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            # Tornado calls streaming_callback on redirect response bodies.
            # But we need streaming to support fetching large files (> RAM avail).
            # Here we working this around by disabling recording the body for redirections.
            # The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
            # when we'll deprecate Tornado<4.3.0.
            # See #27093 and #30431 for details.

            # Use list here to make it writable inside the on_header callback. Simple bool doesn't
            # work here: on_header creates a new local variable instead. This could be avoided in
            # Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
            write_body = [None, False, None]

            def on_header(hdr):
                if write_body[1] is not False and write_body[2] is None:
                    # Try to find out what content type encoding is used if this is a text file
                    write_body[1].parse_line(hdr)  # pylint: disable=no-member
                    if 'Content-Type' in write_body[1]:
                        content_type = write_body[1].get('Content-Type')  # pylint: disable=no-member
                        if not content_type.startswith('text'):
                            write_body[1] = write_body[2] = False
                        else:
                            encoding = 'utf-8'
                            fields = content_type.split(';')
                            for field in fields:
                                if 'encoding' in field:
                                    encoding = field.split('encoding=')[-1]
                            write_body[2] = encoding
                            # We have found our encoding. Stop processing headers.
                            write_body[1] = False

                if write_body[0] is not None:
                    # We already parsed the first line. No need to run the code below again
                    return

                try:
                    hdr = parse_response_start_line(hdr)
                except HTTPInputError:
                    # Not the first line, do nothing
                    return
                write_body[0] = hdr.code not in [301, 302, 303, 307]
                write_body[1] = HTTPHeaders()

            if no_cache:
                result = []

                def on_chunk(chunk):
                    if write_body[0]:
                        if write_body[2]:
                            chunk = chunk.decode(write_body[2])
                        result.append(chunk)
            else:
                dest_tmp = "{0}.part".format(dest)
                # We need an open filehandle to use in the on_chunk callback,
                # that's why we're not using a with clause here.
                destfp = salt.utils.fopen(dest_tmp, 'wb')

                def on_chunk(chunk):
                    if write_body[0]:
                        destfp.write(chunk)

            query = salt.utils.http.query(
                fixed_url,
                stream=True,
                streaming_callback=on_chunk,
                header_callback=on_header,
                username=url_data.username,
                password=url_data.password,
                opts=self.opts,
                **get_kwargs
            )
            if 'handle' not in query:
                raise MinionError('Error: {0} reading {1}'.format(query['error'], url))
            if no_cache:
                if write_body[2]:
                    return six.u('').join(result)
                return six.b('').join(result)
            else:
                destfp.close()
                destfp = None
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #6
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)
        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        else:
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                url_data.netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True))
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                             self.opts.get('keystone.tenant', None),
                                             self.opts.get('keystone.auth_url', None),
                                             self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception as ex:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            passwd_mgr = url_passwd_mgr()
            passwd_mgr.add_password(
                None, fixed_url, url_data.username, url_data.password)
            auth_handler = url_auth_handler(passwd_mgr)
            opener = url_build_opener(auth_handler)
            url_install_opener(opener)
        else:
            fixed_url = url
        try:
            req = requests.get(fixed_url)
            with salt.utils.fopen(dest, 'wb') as destfp:
                destfp.write(req.content)
            return dest
        except HTTPError as ex:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                ex.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[ex.code]))
        except URLError as ex:
            raise MinionError('Error reading {0}: {1}'.format(url, ex.reason))
Beispiel #7
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path {0!r} is not absolute'.format(url_data.path)
                )
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True),
                                    location=self.opts.get('s3.location',
                                                              None))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                             self.opts.get('keystone.tenant', None),
                                             self.opts.get('keystone.auth_url', None),
                                             self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url
        try:
            query = salt.utils.http.query(
                fixed_url,
                stream=True,
                **get_kwargs
            )
            if 'handle' not in query:
                raise MinionError('Error: {0}'.format(query['error']))
            response = query['handle']
            chunk_size = 32 * 1024
            if not no_cache:
                with salt.utils.fopen(dest, 'wb') as destfp:
                    if hasattr(response, 'iter_content'):
                        for chunk in response.iter_content(chunk_size=chunk_size):
                            destfp.write(chunk)
                    else:
                        while True:
                            chunk = response.read(chunk_size)
                            destfp.write(chunk)
                            if len(chunk) < chunk_size:
                                break
                return dest
            else:
                if hasattr(response, 'text'):
                    return response.text
                else:
                    return response['text']
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
Beispiel #8
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path {0!r} is not absolute'.format(url_data.path)
                )
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            if salt.utils.is_windows():
                netloc = salt.utils.sanitize_win_path_string(url_data.netloc)
            else:
                netloc = url_data.netloc
            dest = salt.utils.path_join(
                self.opts['cachedir'],
                'extrn_files',
                saltenv,
                netloc,
                url_data.path
            )
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                             True))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                       self.opts.get('keystone.tenant', None),
                                       self.opts.get('keystone.auth_url', None),
                                       self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url
        try:
            query = salt.utils.http.query(
                fixed_url,
                stream=True,
                username=url_data.username,
                password=url_data.password,
                **get_kwargs
            )
            response = query['handle']
            chunk_size = 32 * 1024
            if not no_cache:
                with salt.utils.fopen(dest, 'wb') as destfp:
                    if hasattr(response, 'iter_content'):
                        for chunk in response.iter_content(chunk_size=chunk_size):
                            destfp.write(chunk)
                    else:
                        while True:
                            chunk = response.buffer.read(chunk_size)
                            destfp.write(chunk)
                            if len(chunk) < chunk_size:
                                break
                return dest
            else:
                if hasattr(response, 'text'):
                    return response.text
                else:
                    return response['text']
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
Beispiel #9
0
    def get_url(self, url, dest, makedirs=False, saltenv='base',
                no_cache=False, cachedir=None):
        '''
        Get a single file from a URL.
        '''
        url_data = urlparse(url)
        url_scheme = url_data.scheme
        url_path = os.path.join(
                url_data.netloc, url_data.path).rstrip(os.sep)

        if url_scheme and url_scheme.lower() in string.ascii_lowercase:
            url_path = ':'.join((url_scheme, url_path))
            url_scheme = 'file'

        if url_scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_path):
                raise CommandExecutionError(
                    'Path \'{0}\' is not absolute'.format(url_path)
                )
            if dest is None:
                with salt.utils.fopen(url_data.path, 'r') as fp_:
                    data = fp_.read()
                return data
            return url_path

        if url_scheme == 'salt':
            result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
            if result and dest is None:
                with salt.utils.fopen(result, 'r') as fp_:
                    data = fp_.read()
                return data
            return result

        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv, cachedir=cachedir)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                def s3_opt(key, default=None):
                    '''Get value of s3.<key> from Minion config or from Pillar'''
                    if 's3.' + key in self.opts:
                        return self.opts['s3.' + key]
                    try:
                        return self.opts['pillar']['s3'][key]
                    except (KeyError, TypeError):
                        return default
                self.utils['s3.query'](method='GET',
                                       bucket=url_data.netloc,
                                       path=url_data.path[1:],
                                       return_bin=False,
                                       local_file=dest,
                                       action=None,
                                       key=s3_opt('key'),
                                       keyid=s3_opt('keyid'),
                                       service_url=s3_opt('service_url'),
                                       verify_ssl=s3_opt('verify_ssl', True),
                                       location=s3_opt('location'))
                return dest
            except Exception as exc:
                raise MinionError(
                    'Could not fetch from {0}. Exception: {1}'.format(url, exc)
                )
        if url_data.scheme == 'ftp':
            try:
                ftp = ftplib.FTP(url_data.hostname)
                ftp.login()
                with salt.utils.fopen(dest, 'wb') as fp_:
                    ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
                return dest
            except Exception as exc:
                raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))

        if url_data.scheme == 'swift':
            try:
                def swift_opt(key, default):
                    '''Get value of <key> from Minion config or from Pillar'''
                    if key in self.opts:
                        return self.opts[key]
                    try:
                        return self.opts['pillar'][key]
                    except (KeyError, TypeError):
                        return default

                swift_conn = SaltSwift(swift_opt('keystone.user', None),
                                       swift_opt('keystone.tenant', None),
                                       swift_opt('keystone.auth_url', None),
                                       swift_opt('keystone.password', None))

                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            netloc = url_data.netloc
            at_sign_pos = netloc.rfind('@')
            if at_sign_pos != -1:
                netloc = netloc[at_sign_pos + 1:]
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            # Tornado calls streaming_callback on redirect response bodies.
            # But we need streaming to support fetching large files (> RAM avail).
            # Here we working this around by disabling recording the body for redirections.
            # The issue is fixed in Tornado 4.3.0 so on_header callback could be removed
            # when we'll deprecate Tornado<4.3.0.
            # See #27093 and #30431 for details.

            # Use list here to make it writable inside the on_header callback. Simple bool doesn't
            # work here: on_header creates a new local variable instead. This could be avoided in
            # Py3 with 'nonlocal' statement. There is no Py2 alternative for this.
            write_body = [False]

            def on_header(hdr):
                try:
                    hdr = parse_response_start_line(hdr)
                except HTTPInputError:
                    # Not the first line, do nothing
                    return
                write_body[0] = hdr.code not in [301, 302, 303, 307]

            if no_cache:
                result = []

                def on_chunk(chunk):
                    if write_body[0]:
                        result.append(chunk)
            else:
                dest_tmp = "{0}.part".format(dest)
                # We need an open filehandle to use in the on_chunk callback,
                # that's why we're not using a with clause here.
                destfp = salt.utils.fopen(dest_tmp, 'wb')

                def on_chunk(chunk):
                    if write_body[0]:
                        destfp.write(chunk)

            query = salt.utils.http.query(
                fixed_url,
                stream=True,
                streaming_callback=on_chunk,
                header_callback=on_header,
                username=url_data.username,
                password=url_data.password,
                opts=self.opts,
                **get_kwargs
            )
            if 'handle' not in query:
                raise MinionError('Error: {0} reading {1}'.format(query['error'], url))
            if no_cache:
                return ''.join(result)
            else:
                destfp.close()
                destfp = None
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #10
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path {0!r} is not absolute'.format(url_data.path)
                )
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=self.opts.get('s3.key', None),
                                    keyid=self.opts.get('s3.keyid', None),
                                    service_url=self.opts.get('s3.service_url',
                                                              None),
                                    verify_ssl=self.opts.get('s3.verify_ssl',
                                                              True),
                                    location=self.opts.get('s3.location',
                                                              None))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(self.opts.get('keystone.user', None),
                                       self.opts.get('keystone.tenant', None),
                                       self.opts.get('keystone.auth_url', None),
                                       self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            if no_cache:
                result = []

                def on_chunk(chunk):
                    result.append(chunk)
            else:
                dest_tmp = "{0}.part".format(dest)
                destfp = salt.utils.fopen(dest_tmp, 'wb')

                def on_chunk(chunk):
                    destfp.write(chunk)

            query = salt.utils.http.query(
                fixed_url,
                stream=True,
                streaming_callback=on_chunk,
                username=url_data.username,
                password=url_data.password,
                **get_kwargs
            )
            if 'handle' not in query:
                raise MinionError('Error: {0}'.format(query['error']))
            if no_cache:
                return ''.join(result)
            else:
                destfp.close()
                destfp = None
                # Can't just do an os.rename() here, this results in a
                # WindowsError being raised when the destination path exists on
                # a Windows machine. Have to remove the file.
                try:
                    os.remove(dest)
                except OSError as exc:
                    if exc.errno != errno.ENOENT:
                        raise MinionError(
                            'Error: Unable to remove {0}: {1}'.format(
                                dest,
                                exc.strerror
                            )
                        )
                os.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #11
0
    def get_url(self, url, dest, makedirs=False, saltenv=u'base',
                no_cache=False, cachedir=None):
        '''
        Get a single file from a URL.
        '''
        url_data = urlparse(url)
        url_scheme = url_data.scheme
        url_path = os.path.join(
                url_data.netloc, url_data.path).rstrip(os.sep)

        # If dest is a directory, rewrite dest with filename
        if dest is not None \
                and (os.path.isdir(dest) or dest.endswith((u'/', u'\\'))):
            if url_data.query or len(url_data.path) > 1 and not url_data.path.endswith(u'/'):
                strpath = url.split(u'/')[-1]
            else:
                strpath = u'index.html'

            if salt.utils.platform.is_windows():
                strpath = salt.utils.sanitize_win_path_string(strpath)

            dest = os.path.join(dest, strpath)

        if url_scheme and url_scheme.lower() in string.ascii_lowercase:
            url_path = u':'.join((url_scheme, url_path))
            url_scheme = u'file'

        if url_scheme in (u'file', u''):
            # Local filesystem
            if not os.path.isabs(url_path):
                raise CommandExecutionError(
                    u'Path \'{0}\' is not absolute'.format(url_path)
                )
            if dest is None:
                with salt.utils.files.fopen(url_path, u'r') as fp_:
                    data = fp_.read()
                return data
            return url_path

        if url_scheme == u'salt':
            result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir)
            if result and dest is None:
                with salt.utils.files.fopen(result, u'r') as fp_:
                    data = fp_.read()
                return data
            return result

        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return u''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv, cachedir=cachedir)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == u's3':
            try:
                def s3_opt(key, default=None):
                    u'''Get value of s3.<key> from Minion config or from Pillar'''
                    if u's3.' + key in self.opts:
                        return self.opts[u's3.' + key]
                    try:
                        return self.opts[u'pillar'][u's3'][key]
                    except (KeyError, TypeError):
                        return default
                self.utils[u's3.query'](method=u'GET',
                                       bucket=url_data.netloc,
                                       path=url_data.path[1:],
                                       return_bin=False,
                                       local_file=dest,
                                       action=None,
                                       key=s3_opt(u'key'),
                                       keyid=s3_opt(u'keyid'),
                                       service_url=s3_opt(u'service_url'),
                                       verify_ssl=s3_opt(u'verify_ssl', True),
                                       location=s3_opt(u'location'),
                                       path_style=s3_opt(u'path_style', False),
                                       https_enable=s3_opt(u'https_enable', True))
                return dest
            except Exception as exc:
                raise MinionError(
                    u'Could not fetch from {0}. Exception: {1}'.format(url, exc)
                )
        if url_data.scheme == u'ftp':
            try:
                ftp = ftplib.FTP()
                ftp.connect(url_data.hostname, url_data.port)
                ftp.login(url_data.username, url_data.password)
                with salt.utils.files.fopen(dest, u'wb') as fp_:
                    ftp.retrbinary(u'RETR {0}'.format(url_data.path), fp_.write)
                ftp.quit()
                return dest
            except Exception as exc:
                raise MinionError(u'Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))

        if url_data.scheme == u'swift':
            try:
                def swift_opt(key, default):
                    '''
                    Get value of <key> from Minion config or from Pillar
                    '''
                    if key in self.opts:
                        return self.opts[key]
                    try:
                        return self.opts[u'pillar'][key]
                    except (KeyError, TypeError):
                        return default

                swift_conn = SaltSwift(swift_opt(u'keystone.user', None),
                                       swift_opt(u'keystone.tenant', None),
                                       swift_opt(u'keystone.auth_url', None),
                                       swift_opt(u'keystone.password', None))

                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError(u'Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in (u'http', u'https'):
            netloc = url_data.netloc
            at_sign_pos = netloc.rfind(u'@')
            if at_sign_pos != -1:
                netloc = netloc[at_sign_pos + 1:]
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs[u'auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            # Tornado calls streaming_callback on redirect response bodies.
            # But we need streaming to support fetching large files (> RAM
            # avail). Here we are working around this by disabling recording
            # the body for redirections. The issue is fixed in Tornado 4.3.0
            # so on_header callback could be removed when we'll deprecate
            # Tornado<4.3.0. See #27093 and #30431 for details.

            # Use list here to make it writable inside the on_header callback.
            # Simple bool doesn't work here: on_header creates a new local
            # variable instead. This could be avoided in Py3 with 'nonlocal'
            # statement. There is no Py2 alternative for this.
            #
            # write_body[0] is used by the on_chunk callback to tell it whether
            #   or not we need to write the body of the request to disk. For
            #   30x redirects we set this to False because we don't want to
            #   write the contents to disk, as we will need to wait until we
            #   get to the redirected URL.
            #
            # write_body[1] will contain a tornado.httputil.HTTPHeaders
            #   instance that we will use to parse each header line. We
            #   initialize this to False, and after we parse the status line we
            #   will replace it with the HTTPHeaders instance. If/when we have
            #   found the encoding used in the request, we set this value to
            #   False to signify that we are done parsing.
            #
            # write_body[2] is where the encoding will be stored
            write_body = [None, False, None]

            def on_header(hdr):
                if write_body[1] is not False and write_body[2] is None:
                    if not hdr.strip() and 'Content-Type' not in write_body[1]:
                        # We've reached the end of the headers and not yet
                        # found the Content-Type. Reset the values we're
                        # tracking so that we properly follow the redirect.
                        write_body[0] = None
                        write_body[1] = False
                        return
                    # Try to find out what content type encoding is used if
                    # this is a text file
                    write_body[1].parse_line(hdr)  # pylint: disable=no-member
                    if u'Content-Type' in write_body[1]:
                        content_type = write_body[1].get(u'Content-Type')  # pylint: disable=no-member
                        if not content_type.startswith(u'text'):
                            write_body[1] = write_body[2] = False
                        else:
                            encoding = u'utf-8'
                            fields = content_type.split(u';')
                            for field in fields:
                                if u'encoding' in field:
                                    encoding = field.split(u'encoding=')[-1]
                            write_body[2] = encoding
                            # We have found our encoding. Stop processing headers.
                            write_body[1] = False

                        # If write_body[0] is False, this means that this
                        # header is a 30x redirect, so we need to reset
                        # write_body[0] to None so that we parse the HTTP
                        # status code from the redirect target.
                        if write_body[0] is write_body[1] is False:
                            write_body[0] = None

                # Check the status line of the HTTP request
                if write_body[0] is None:
                    try:
                        hdr = parse_response_start_line(hdr)
                    except HTTPInputError:
                        # Not the first line, do nothing
                        return
                    write_body[0] = hdr.code not in [301, 302, 303, 307]
                    write_body[1] = HTTPHeaders()

            if no_cache:
                result = []

                def on_chunk(chunk):
                    if write_body[0]:
                        if write_body[2]:
                            chunk = chunk.decode(write_body[2])
                        result.append(chunk)
            else:
                dest_tmp = u"{0}.part".format(dest)
                # We need an open filehandle to use in the on_chunk callback,
                # that's why we're not using a with clause here.
                destfp = salt.utils.files.fopen(dest_tmp, u'wb')  # pylint: disable=resource-leakage

                def on_chunk(chunk):
                    if write_body[0]:
                        destfp.write(chunk)

            query = salt.utils.http.query(
                fixed_url,
                stream=True,
                streaming_callback=on_chunk,
                header_callback=on_header,
                username=url_data.username,
                password=url_data.password,
                opts=self.opts,
                **get_kwargs
            )
            if u'handle' not in query:
                raise MinionError(u'Error: {0} reading {1}'.format(query[u'error'], url))
            if no_cache:
                if write_body[2]:
                    return u''.join(result)
                return six.b(u'').join(result)
            else:
                destfp.close()
                destfp = None
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError(u'HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError(u'Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #12
0
    def get_url(self, url, dest, makedirs=False, saltenv='base', env=None, no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.'
            )
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path \'{0}\' is not absolute'.format(url_data.path)
                )
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                def s3_opt(key, default=None):
                    '''Get value of s3.<key> from Minion config or from Pillar'''
                    if 's3.' + key in self.opts:
                        return self.opts['s3.' + key]
                    try:
                        return self.opts['pillar']['s3'][key]
                    except (KeyError, TypeError):
                        return default
                salt.utils.s3.query(method='GET',
                                    bucket=url_data.netloc,
                                    path=url_data.path[1:],
                                    return_bin=False,
                                    local_file=dest,
                                    action=None,
                                    key=s3_opt('key'),
                                    keyid=s3_opt('keyid'),
                                    service_url=s3_opt('service_url'),
                                    verify_ssl=s3_opt('verify_ssl', True),
                                    location=s3_opt('location'))
                return dest
            except Exception as exc:
                raise MinionError('Could not fetch from {0}. Exception: {1}'.format(url, exc))
        if url_data.scheme == 'ftp':
            try:
                ftp = ftplib.FTP(url_data.hostname)
                ftp.login()
                with salt.utils.fopen(dest, 'wb') as fp_:
                    ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write)
                return dest
            except Exception as exc:
                raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc))

        if url_data.scheme == 'swift':
            try:
                def swift_opt(key, default):
                    '''Get value of <key> from Minion config or from Pillar'''
                    if key in self.opts:
                        return self.opts[key]
                    try:
                        return self.opts['pillar'][key]
                    except (KeyError, TypeError):
                        return default

                swift_conn = SaltSwift(swift_opt('keystone.user', None),
                                       swift_opt('keystone.tenant', None),
                                       swift_opt('keystone.auth_url', None),
                                       swift_opt('keystone.password', None))

                swift_conn.get_object(url_data.netloc,
                                      url_data.path[1:],
                                      dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            netloc = url_data.netloc
            at_sign_pos = netloc.rfind('@')
            if at_sign_pos != -1:
                netloc = netloc[at_sign_pos + 1:]
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path,
                 url_data.params, url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            query = salt.utils.http.query(
                fixed_url,
                text=True,
                username=url_data.username,
                password=url_data.password,
                **get_kwargs
            )
            if 'text' not in query:
                raise MinionError('Error: {0}'.format(query['error']))
            if no_cache:
                return query['body']
            else:
                dest_tmp = "{0}.part".format(dest)
                with salt.utils.fopen(dest_tmp, 'wb') as destfp:
                    destfp.write(query['body'])
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code,
                url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()
Beispiel #13
0
    def get_url(self,
                url,
                dest,
                makedirs=False,
                saltenv='base',
                env=None,
                no_cache=False):
        '''
        Get a single file from a URL.
        '''
        if env is not None:
            salt.utils.warn_until(
                'Boron',
                'Passing a salt environment should be done using \'saltenv\' '
                'not \'env\'. This functionality will be removed in Salt '
                'Boron.')
            # Backwards compatibility
            saltenv = env

        url_data = urlparse(url)

        if url_data.scheme in ('file', ''):
            # Local filesystem
            if not os.path.isabs(url_data.path):
                raise CommandExecutionError(
                    'Path \'{0}\' is not absolute'.format(url_data.path))
            return url_data.path

        if url_data.scheme == 'salt':
            return self.get_file(url, dest, makedirs, saltenv)
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    os.makedirs(destdir)
                else:
                    return ''
        elif not no_cache:
            dest = self._extrn_path(url, saltenv)
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)

        if url_data.scheme == 's3':
            try:
                salt.utils.s3.query(
                    method='GET',
                    bucket=url_data.netloc,
                    path=url_data.path[1:],
                    return_bin=False,
                    local_file=dest,
                    action=None,
                    key=self.opts.get('s3.key', None),
                    keyid=self.opts.get('s3.keyid', None),
                    service_url=self.opts.get('s3.service_url', None),
                    verify_ssl=self.opts.get('s3.verify_ssl', True),
                    location=self.opts.get('s3.location', None))
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        if url_data.scheme == 'swift':
            try:
                swift_conn = SaltSwift(
                    self.opts.get('keystone.user', None),
                    self.opts.get('keystone.tenant', None),
                    self.opts.get('keystone.auth_url', None),
                    self.opts.get('keystone.password', None))
                swift_conn.get_object(url_data.netloc, url_data.path[1:], dest)
                return dest
            except Exception:
                raise MinionError('Could not fetch from {0}'.format(url))

        get_kwargs = {}
        if url_data.username is not None \
                and url_data.scheme in ('http', 'https'):
            _, netloc = url_data.netloc.split('@', 1)
            fixed_url = urlunparse(
                (url_data.scheme, netloc, url_data.path, url_data.params,
                 url_data.query, url_data.fragment))
            get_kwargs['auth'] = (url_data.username, url_data.password)
        else:
            fixed_url = url

        destfp = None
        try:
            if no_cache:
                result = []

                def on_chunk(chunk):
                    result.append(chunk)
            else:
                dest_tmp = "{0}.part".format(dest)
                destfp = salt.utils.fopen(dest_tmp, 'wb')

                def on_chunk(chunk):
                    destfp.write(chunk)

            query = salt.utils.http.query(fixed_url,
                                          stream=True,
                                          streaming_callback=on_chunk,
                                          username=url_data.username,
                                          password=url_data.password,
                                          **get_kwargs)

            if 'handle' not in query:
                raise MinionError('Error: {0}'.format(query['error']))

            try:
                content_length = int(query['handle'].headers['Content-Length'])
            except (AttributeError, KeyError, ValueError):
                # Shouldn't happen but don't let this raise an exception.
                # Instead, just don't do content length checking below.
                log.warning(
                    'No Content-Length header in HTTP response from fetch of '
                    '{0}, or Content-Length is non-numeric'.format(fixed_url))
                content_length = None

            if no_cache:
                content = ''.join(result)
                if content_length is not None \
                        and len(content) > content_length:
                    return content[-content_length:]
                else:
                    return content
            else:
                destfp.close()
                destfp = None
                dest_tmp_size = os.path.getsize(dest_tmp)
                if content_length is not None \
                        and dest_tmp_size > content_length:
                    log.warning(
                        'Size of file downloaded from {0} ({1}) does not '
                        'match the Content-Length ({2}). This is probably due '
                        'to an upstream bug in tornado '
                        '(https://github.com/tornadoweb/tornado/issues/1518). '
                        'Re-writing the file to correct this.'.format(
                            fixed_url, dest_tmp_size, content_length))
                    dest_tmp_bak = dest_tmp + '.bak'
                    salt.utils.files.rename(dest_tmp, dest_tmp_bak)
                    with salt.utils.fopen(dest_tmp_bak, 'rb') as fp_bak:
                        fp_bak.seek(dest_tmp_size - content_length)
                        with salt.utils.fopen(dest_tmp, 'wb') as fp_new:
                            while True:
                                chunk = fp_bak.read(
                                    self.opts['file_buffer_size'])
                                if not chunk:
                                    break
                                fp_new.write(chunk)
                    os.remove(dest_tmp_bak)
                salt.utils.files.rename(dest_tmp, dest)
                return dest
        except HTTPError as exc:
            raise MinionError('HTTP error {0} reading {1}: {3}'.format(
                exc.code, url,
                *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code]))
        except URLError as exc:
            raise MinionError('Error reading {0}: {1}'.format(url, exc.reason))
        finally:
            if destfp is not None:
                destfp.close()