Example #1
0
def _download_image(image_info):
    starttime = time.time()
    resp = None
    for url in image_info['urls']:
        try:
            LOG.info("Attempting to download image from {0}".format(url))
            resp = _request_url(image_info, url)
        except errors.ImageDownloadError as e:
            failtime = time.time() - starttime
            log_msg = ('Image download failed. URL: {0}; time: {1} seconds. '
                       'Error: {2}')
            LOG.warning(log_msg.format(url, failtime, e.details))
            continue
        else:
            break
    if resp is None:
        msg = 'Image download failed for all URLs.'
        raise errors.ImageDownloadError(image_info['id'], msg)

    image_location = _image_location(image_info)
    with open(image_location, 'wb') as f:
        try:
            for chunk in resp.iter_content(IMAGE_CHUNK_SIZE):
                f.write(chunk)
        except Exception as e:
            msg = 'Unable to write image to {0}. Error: {1}'.format(
                image_location, str(e))
            raise errors.ImageDownloadError(image_info['id'], msg)

    totaltime = time.time() - starttime
    LOG.info("Image downloaded from {0} in {1} seconds".format(
        image_location, totaltime))

    if not _verify_image(image_info, image_location):
        raise errors.ImageChecksumError(image_info['id'])
Example #2
0
def _fetch_checksum(checksum, image_info):
    """Fetch checksum from remote location, if needed."""
    if not (checksum.startswith('http://') or checksum.startswith('https://')):
        # Not a remote checksum, return as it is.
        return checksum

    LOG.debug('Downloading checksums file from %s', checksum)
    resp = _download_with_proxy(image_info, checksum, checksum).text
    lines = [line.strip() for line in resp.split('\n') if line.strip()]
    if not lines:
        raise errors.ImageDownloadError(checksum, "Empty checksum file")
    elif len(lines) == 1:
        # Special case - checksums file with only the checksum itself
        if ' ' not in lines[0]:
            return lines[0]

    # FIXME(dtantsur): can we assume the same name for all images?
    expected_fname = os.path.basename(
        urlparse.urlparse(image_info['urls'][0]).path)
    for line in lines:
        checksum, fname = line.strip().split(None, 1)
        # The star symbol designates binary mode, which is the same as text
        # mode on GNU systems.
        if fname.strip().lstrip('*') == expected_fname:
            return checksum.strip()

    raise errors.ImageDownloadError(
        checksum, "Checksum file does not contain name %s" % expected_fname)
Example #3
0
def _download_image(image_info):
    starttime = time.time()
    resp = None
    for url in image_info['urls']:
        try:
            LOG.info("Attempting to download image from {0}".format(url))
            resp = _request_url(image_info, url)
        except errors.ImageDownloadError:
            failtime = time.time() - starttime
            log_msg = "Image download failed. URL: {0}; time: {1} seconds"
            LOG.warning(log_msg.format(url, failtime))
            continue
        else:
            break
    if resp is None:
        raise errors.ImageDownloadError(image_info['id'])

    image_location = _image_location(image_info)
    with open(image_location, 'wb') as f:
        try:
            for chunk in resp.iter_content(1024 * 1024):
                f.write(chunk)
        except Exception:
            raise errors.ImageDownloadError(image_info['id'])

    totaltime = time.time() - starttime
    LOG.info("Image downloaded from {0} in {1} seconds".format(
        image_location, totaltime))

    if not _verify_image(image_info, image_location):
        raise errors.ImageChecksumError(image_info['id'])
Example #4
0
def _request_url(image_info, url):
    resp = requests.get(url, stream=True)
    if resp.status_code != 200:
        msg = ('Received status code {0} from {1}, expected 200. Response '
               'body: {2}').format(resp.status_code, url, resp.text)
        raise errors.ImageDownloadError(image_info['id'], msg)
    return resp
Example #5
0
    def _stream_raw_image_onto_device(self, image_info, device):
        """Streams raw image data to specified local device.

        :param image_info: Image information dictionary.
        :param device: The disk name, as a string, on which to store the
                       image.  Example: '/dev/sda'

        :raises: ImageDownloadError if the image download encounters an error.
        :raises: ImageChecksumError if the checksum of the local image does not
             match the checksum as reported by glance in image_info.
        """
        starttime = time.time()
        image_download = ImageDownload(image_info, time_obj=starttime)

        with open(device, 'wb+') as f:
            try:
                for chunk in image_download:
                    f.write(chunk)
            except Exception as e:
                msg = 'Unable to write image to device {0}. Error: {1}'.format(
                      device, str(e))
                raise errors.ImageDownloadError(image_info['id'], msg)

        totaltime = time.time() - starttime
        LOG.info("Image streamed onto device {0} in {1} "
                 "seconds".format(device, totaltime))
        # Verify if the checksum of the streamed image is correct
        _verify_image(image_info, device, image_download.md5sum())
Example #6
0
def _download_with_proxy(image_info, url, image_id):
    """Opens a download stream for the given URL.

    :param image_info: Image information dictionary.
    :param url: The URL string to request the image from.
    :param image_id: Image ID or URL for logging.

    :raises: ImageDownloadError if the download stream was not started
             properly.
    """
    no_proxy = image_info.get('no_proxy')
    if no_proxy:
        os.environ['no_proxy'] = no_proxy
    proxies = image_info.get('proxies', {})
    verify, cert = utils.get_ssl_client_options(CONF)
    resp = requests.get(url,
                        stream=True,
                        proxies=proxies,
                        verify=verify,
                        cert=cert)
    if resp.status_code != 200:
        msg = ('Received status code {} from {}, expected 200. Response '
               'body: {}').format(resp.status_code, url, resp.text)
        raise errors.ImageDownloadError(image_id, msg)
    return resp
Example #7
0
def _download_image(image_info):
    """Downloads the specified image to the local file system.

    :param image_info: Image information dictionary.
    :raises: ImageDownloadError if the image download fails for any reason.
    :raises: ImageChecksumError if the downloaded image's checksum does not
             match the one reported in image_info.
    """
    starttime = time.time()
    image_location = _image_location(image_info)
    image_download = ImageDownload(image_info, time_obj=starttime)

    with open(image_location, 'wb') as f:
        try:
            for chunk in image_download:
                f.write(chunk)
        except Exception as e:
            msg = 'Unable to write image to {0}. Error: {1}'.format(
                image_location, str(e))
            raise errors.ImageDownloadError(image_info['id'], msg)

    totaltime = time.time() - starttime
    LOG.info("Image downloaded from {0} in {1} seconds".format(image_location,
                                                               totaltime))
    _verify_image(image_info, image_location, image_download.md5sum())
Example #8
0
    def __iter__(self):
        """Downloads and returns the next chunk of the image.

        :returns: A chunk of the image. Size of chunk is IMAGE_CHUNK_SIZE
                  which is a constant in this module.
        """
        self._last_chunk_time = None
        for chunk in self._request.iter_content(IMAGE_CHUNK_SIZE):
            # Per requests forum posts/discussions, iter_content should
            # periodically yield to the caller for the client to do things
            # like stopwatch and potentially interrupt the download.
            # While this seems weird and doesn't exactly seem to match the
            # patterns in requests and urllib3, it does appear to be the
            # case. Field testing in environments where TCP sockets were
            # discovered in a read hanged state were navigated with
            # this code.
            if chunk:
                self._last_chunk_time = time.time()
                self._hash_algo.update(chunk)
                yield chunk
            elif (time.time() - self._last_chunk_time >
                  CONF.image_download_connection_timeout):
                LOG.error('Timeout reached waiting for a chunk of data from '
                          'a remote server.')
                raise errors.ImageDownloadError(
                    self._image_info['id'],
                    'Timed out reading next chunk from webserver')
Example #9
0
    def _stream_raw_image_onto_device(self, image_info, device):
        """Streams raw image data to specified local device.

        :param image_info: Image information dictionary.
        :param device: The disk name, as a string, on which to store the
                       image.  Example: '/dev/sda'

        :raises: ImageDownloadError if the image download encounters an error.
        :raises: ImageChecksumError if the checksum of the local image does not
             match the checksum as reported by glance in image_info.
        """
        starttime = time.time()
        image_download = ImageDownload(image_info, time_obj=starttime)

        with open(device, 'wb+') as f:
            try:
                for chunk in image_download:
                    f.write(chunk)
            except Exception as e:
                msg = 'Unable to write image to device {}. Error: {}'.format(
                    device, str(e))
                raise errors.ImageDownloadError(image_info['id'], msg)

        totaltime = time.time() - starttime
        LOG.info("Image streamed onto device {} in {} "
                 "seconds".format(device, totaltime))
        # Verify if the checksum of the streamed image is correct
        image_download.verify_image(device)
        # Fix any gpt partition
        try:
            disk_utils.fix_gpt_partition(device, node_uuid=None)
        except exception.InstanceDeployFailure:
            # Note: the catch internal to the helper method logs any errors.
            pass
Example #10
0
    def __init__(self, image_info, time_obj=None):
        """Initialize an instance of the ImageDownload class.

        Trys each URL in image_info successively until a URL returns a
        successful request code. Once the object is initialized, the user may
        retrieve chunks of the image through the standard python iterator
        interface until either the image is fully downloaded, or an error is
        encountered.

        :param image_info: Image information dictionary.
        :param time_obj: Optional time object to indicate when the image
                         download began. Defaults to None. If None, then
                         time.time() will be used to find the start time of
                         the download.

        :raises: ImageDownloadError if starting the image download fails for
                 any reason.
        """
        self._time = time_obj or time.time()
        self._image_info = image_info
        self._request = None

        # Determine the hash algorithm and value will be used for calculation
        # and verification, fallback to md5 if algorithm is not set or not
        # supported.
        algo = image_info.get('os_hash_algo')
        if algo and algo in hashlib.algorithms_available:
            self._hash_algo = hashlib.new(algo)
            self._expected_hash_value = image_info.get('os_hash_value')
        else:
            self._hash_algo = hashlib.md5()
            self._expected_hash_value = image_info['checksum']

        self._expected_hash_value = _fetch_checksum(self._expected_hash_value,
                                                    image_info)

        details = []
        for url in image_info['urls']:
            try:
                LOG.info("Attempting to download image from {}".format(url))
                self._request = _download_with_proxy(image_info, url,
                                                     image_info['id'])
            except errors.ImageDownloadError as e:
                failtime = time.time() - self._time
                log_msg = ('URL: {}; time: {} '
                           'seconds. Error: {}').format(
                               url, failtime, e.secondary_message)
                LOG.warning(log_msg)
                details.append(log_msg)
                continue
            else:
                break
        else:
            details = '\n '.join(details)
            raise errors.ImageDownloadError(image_info['id'], details)
 def _download_file(self, image_info, url):
     no_proxy = image_info.get('no_proxy')
     if no_proxy:
         os.environ['no_proxy'] = no_proxy
     proxies = image_info.get('proxies', {})
     resp = requests.get(url, stream=True, proxies=proxies)
     if resp.status_code != 200:
         msg = ('Received status code {0} from {1}, expected 200. Response '
                'body: {2}').format(resp.status_code, url, resp.text)
         raise errors.ImageDownloadError(image_info['id'], msg)
     return resp
Example #12
0
def _download_with_proxy(image_info, url, image_id):
    """Opens a download stream for the given URL.

    :param image_info: Image information dictionary.
    :param url: The URL string to request the image from.
    :param image_id: Image ID or URL for logging.

    :raises: ImageDownloadError if the download stream was not started
             properly.
    """
    no_proxy = image_info.get('no_proxy')
    if no_proxy:
        os.environ['no_proxy'] = no_proxy
    proxies = image_info.get('proxies', {})
    verify, cert = utils.get_ssl_client_options(CONF)
    resp = None
    for attempt in range(CONF.image_download_connection_retries + 1):
        try:
            # NOTE(TheJulia) The get request below does the following:
            # * Performs dns lookups, if necessary
            # * Opens the TCP socket to the remote host
            # * Negotiates TLS, if applicable
            # * Checks cert validity, if necessary, which may be
            #   more tcp socket connections.
            # * Issues the get request and then returns back to the caller the
            #   handler which is used to stream the data into the agent.
            # While this all may be at risk of transitory interrupts, most of
            # these socket will have timeouts applied to them, although not
            # exactly just as the timeout value exists. The risk in transitory
            # failure is more so once we've started the download and we are
            # processing the incoming data.
            resp = requests.get(url,
                                stream=True,
                                proxies=proxies,
                                verify=verify,
                                cert=cert,
                                timeout=CONF.image_download_connection_timeout)
            if resp.status_code != 200:
                msg = ('Received status code {} from {}, expected 200. '
                       'Response body: {}').format(resp.status_code, url,
                                                   resp.text)
                raise errors.ImageDownloadError(image_id, msg)
        except (errors.ImageDownloadError, requests.RequestException) as e:
            if (attempt == CONF.image_download_connection_retries
                    # NOTE(dtantsur): do not retry 4xx status codes
                    or (resp and resp.status_code < 500)):
                raise
            else:
                LOG.warning('Unable to connect to %s, retrying. Error: %s',
                            url, e)
                time.sleep(CONF.image_download_connection_retry_interval)
        else:
            break
    return resp
Example #13
0
    def _stream_raw_image_onto_device(self, image_info, device):
        """Streams raw image data to specified local device.

        :param image_info: Image information dictionary.
        :param device: The disk name, as a string, on which to store the
                       image.  Example: '/dev/sda'

        :raises: ImageDownloadError if the image download encounters an error.
        :raises: ImageChecksumError if the checksum of the local image does not
             match the checksum as reported by glance in image_info.
        """
        starttime = time.time()
        total_retries = CONF.image_download_connection_retries
        for attempt in range(total_retries + 1):
            try:
                image_download = ImageDownload(image_info, time_obj=starttime)

                with open(device, 'wb+') as f:
                    try:
                        for chunk in image_download:
                            f.write(chunk)
                    except Exception as e:
                        msg = ('Unable to write image to device {}. '
                               'Error: {}').format(device, str(e))
                        raise errors.ImageDownloadError(image_info['id'], msg)
            except errors.ImageDownloadError as e:
                if attempt == CONF.image_download_connection_retries:
                    raise
                else:
                    LOG.warning('Image download failed, %(attempt)s of '
                                '%(total)s: %(error)s',
                                {'attempt': attempt,
                                 'total': total_retries,
                                 'error': e})
                    time.sleep(CONF.image_download_connection_retry_interval)
            else:
                break

        totaltime = time.time() - starttime
        LOG.info("Image streamed onto device {} in {} "
                 "seconds".format(device, totaltime))
        # Verify if the checksum of the streamed image is correct
        image_download.verify_image(device)
        # Fix any gpt partition
        try:
            disk_utils.fix_gpt_partition(device, node_uuid=None)
        except exception.InstanceDeployFailure:
            # Note: the catch internal to the helper method logs any errors.
            pass
        # Fix the root partition UUID
        root_uuid = disk_utils.block_uuid(device)
        LOG.info("{} UUID is now {}".format(device, root_uuid))
        self.partition_uuids['root uuid'] = root_uuid
Example #14
0
    def _stream_raw_image_onto_device(self, image_info, device):
        """Streams raw image data to specified local device.

        :param image_info: Image information dictionary.
        :param device: The disk name, as a string, on which to store the
                       image.  Example: '/dev/sda'

        :raises: ImageDownloadError if the image download encounters an error.
        :raises: ImageChecksumError if the checksum of the local image does not
             match the checksum as reported by glance in image_info.
        """
        starttime = time.time()
        total_retries = CONF.image_download_connection_retries
        for attempt in range(total_retries + 1):
            try:
                image_download = ImageDownload(image_info, time_obj=starttime)

                with open(device, 'wb+') as f:
                    try:
                        for chunk in image_download:
                            f.write(chunk)
                    except Exception as e:
                        msg = ('Unable to write image to device {}. '
                               'Error: {}').format(device, str(e))
                        raise errors.ImageDownloadError(image_info['id'], msg)
            except errors.ImageDownloadError as e:
                if attempt == CONF.image_download_connection_retries:
                    raise
                else:
                    LOG.warning(
                        'Image download failed, %(attempt)s of '
                        '%(total)s: %(error)s', {
                            'attempt': attempt,
                            'total': total_retries,
                            'error': e
                        })
                    time.sleep(CONF.image_download_connection_retry_interval)
            else:
                break

        totaltime = time.time() - starttime
        LOG.info("Image streamed onto device {} in {} "
                 "seconds".format(device, totaltime))
        # Verify if the checksum of the streamed image is correct
        image_download.verify_image(device)
def _download_image(image_info):
    starttime = time.time()
    image_location = _image_location(image_info)
    image_download = ImageDownload(image_info, time_obj=starttime)

    with open(image_location, 'wb') as f:
        try:
            for chunk in image_download:
                f.write(chunk)
        except Exception as e:
            msg = 'Unable to write image to {0}. Error: {1}'.format(
                image_location, str(e))
            raise errors.ImageDownloadError(image_info['id'], msg)

    totaltime = time.time() - starttime
    LOG.info("Image downloaded from {0} in {1} seconds".format(
        image_location, totaltime))
    _verify_image(image_info, image_location, image_download.md5sum())
    def _stream_raw_image_onto_device(self, image_info, device):
        starttime = time.time()
        image_download = ImageDownload(image_info, time_obj=starttime)

        with open(device, 'wb+') as f:
            try:
                for chunk in image_download:
                    f.write(chunk)
            except Exception as e:
                msg = 'Unable to write image to device {0}. Error: {1}'.format(
                    device, str(e))
                raise errors.ImageDownloadError(image_info['id'], msg)

        totaltime = time.time() - starttime
        LOG.info("Image streamed onto device {0} in {1} "
                 "seconds".format(device, totaltime))
        # Verify if the checksum of the streamed image is correct
        _verify_image(image_info, device, image_download.md5sum())
Example #17
0
    def _download_file(self, image_info, url):
        """Opens a download stream for the given URL.

        :param image_info: Image information dictionary.
        :param url: The URL string to request the image from.

        :raises: ImageDownloadError if the download stream was not started
                 properly.
        """
        no_proxy = image_info.get('no_proxy')
        if no_proxy:
            os.environ['no_proxy'] = no_proxy
        proxies = image_info.get('proxies', {})
        resp = requests.get(url, stream=True, proxies=proxies)
        if resp.status_code != 200:
            msg = ('Received status code {0} from {1}, expected 200. Response '
                   'body: {2}').format(resp.status_code, url, resp.text)
            raise errors.ImageDownloadError(image_info['id'], msg)
        return resp
    def __init__(self, image_info, time_obj=None):
        self._md5checksum = hashlib.md5()
        self._time = time_obj or time.time()
        self._request = None

        for url in image_info['urls']:
            try:
                LOG.info("Attempting to download image from {0}".format(url))
                self._request = self._download_file(image_info, url)
            except errors.ImageDownloadError as e:
                failtime = time.time() - self._time
                log_msg = ('Image download failed. URL: {0}; time: {1} '
                           'seconds. Error: {2}')
                LOG.warning(log_msg.format(url, failtime, e.details))
                continue
            else:
                break
        else:
            msg = 'Image download failed for all URLs.'
            raise errors.ImageDownloadError(image_info['id'], msg)
Example #19
0
    def __init__(self, image_info, time_obj=None):
        """Initialize an instance of the ImageDownload class.

        Trys each URL in image_info successively until a URL returns a
        successful request code. Once the object is initialized, the user may
        retrieve chunks of the image through the standard python iterator
        interface until either the image is fully downloaded, or an error is
        encountered.

        :param image_info: Image information dictionary.
        :param time_obj: Optional time object to indicate when the image
                         download began. Defaults to None. If None, then
                         time.time() will be used to find the start time of
                         the download.

        :raises: ImageDownloadError if starting the image download fails for
                 any reason.
        """
        self._md5checksum = hashlib.md5()
        self._time = time_obj or time.time()
        self._request = None
        details = []
        for url in image_info['urls']:
            try:
                LOG.info("Attempting to download image from {}".format(url))
                self._request = self._download_file(image_info, url)
            except errors.ImageDownloadError as e:
                failtime = time.time() - self._time
                log_msg = ('URL: {}; time: {} '
                           'seconds. Error: {}').format(
                               url, failtime, e.details)
                LOG.warning('Image download failed. %s', log_msg)
                details += log_msg
                continue
            else:
                break
        else:
            details = '/n'.join(details)
            msg = ('Image download failed for all URLs with following errors: '
                   '{}'.format(details))
            raise errors.ImageDownloadError(image_info['id'], msg)
Example #20
0
def _download_image(image_info):
    """Downloads the specified image to the local file system.

    :param image_info: Image information dictionary.
    :raises: ImageDownloadError if the image download fails for any reason.
    :raises: ImageChecksumError if the downloaded image's checksum does not
             match the one reported in image_info.
    """
    starttime = time.time()
    image_location = _image_location(image_info)
    for attempt in range(CONF.image_download_connection_retries + 1):
        try:
            image_download = ImageDownload(image_info, time_obj=starttime)

            with open(image_location, 'wb') as f:
                try:
                    for chunk in image_download:
                        f.write(chunk)
                except Exception as e:
                    msg = 'Unable to write image to {}. Error: {}'.format(
                        image_location, str(e))
                    raise errors.ImageDownloadError(image_info['id'], msg)
        except errors.ImageDownloadError as e:
            if attempt == CONF.image_download_connection_retries:
                raise
            else:
                LOG.warning(
                    'Image download failed, %(attempt)s of %(total)s: '
                    '%(error)s', {
                        'attempt': attempt,
                        'total': CONF.image_download_connection_retries,
                        'error': e
                    })
                time.sleep(CONF.image_download_connection_retry_interval)
        else:
            break

    totaltime = time.time() - starttime
    LOG.info("Image downloaded from {} in {} seconds".format(
        image_location, totaltime))
    image_download.verify_image(image_location)
Example #21
0
 def test_error_classes(self):
     cases = [
         (errors.InvalidContentError(DETAILS), SAME_DETAILS),
         (errors.NotFound(), SAME_CL_DETAILS),
         (errors.CommandExecutionError(DETAILS), SAME_DETAILS),
         (errors.InvalidCommandError(DETAILS), SAME_DETAILS),
         (errors.InvalidCommandParamsError(DETAILS), SAME_DETAILS),
         (errors.RequestedObjectNotFoundError('type_descr',
                                              'obj_id'), DIFF_CL_DETAILS),
         (errors.IronicAPIError(DETAILS), SAME_DETAILS),
         (errors.HeartbeatError(DETAILS), SAME_DETAILS),
         (errors.LookupNodeError(DETAILS), SAME_DETAILS),
         (errors.LookupAgentIPError(DETAILS), SAME_DETAILS),
         (errors.LookupAgentInterfaceError(DETAILS), SAME_DETAILS),
         (errors.ImageDownloadError('image_id', DETAILS), DIFF_CL_DETAILS),
         (errors.ImageChecksumError('image_id', '/foo/image_id',
                                    'incorrect',
                                    'correct'), DIFF_CL_DETAILS),
         (errors.ImageWriteError('device', 'exit_code', 'stdout',
                                 'stderr'), DIFF_CL_DETAILS),
         (errors.ConfigDriveTooLargeError('filename',
                                          'filesize'), DIFF_CL_DETAILS),
         (errors.ConfigDriveWriteError('device', 'exit_code', 'stdout',
                                       'stderr'), DIFF_CL_DETAILS),
         (errors.SystemRebootError('exit_code', 'stdout',
                                   'stderr'), DIFF_CL_DETAILS),
         (errors.BlockDeviceEraseError(DETAILS), SAME_DETAILS),
         (errors.BlockDeviceError(DETAILS), SAME_DETAILS),
         (errors.VirtualMediaBootError(DETAILS), SAME_DETAILS),
         (errors.UnknownNodeError(), DEFAULT_DETAILS),
         (errors.UnknownNodeError(DETAILS), SAME_DETAILS),
         (errors.HardwareManagerNotFound(), DEFAULT_DETAILS),
         (errors.HardwareManagerNotFound(DETAILS), SAME_DETAILS),
         (errors.HardwareManagerMethodNotFound('method'), DIFF_CL_DETAILS),
         (errors.IncompatibleHardwareMethodError(), DEFAULT_DETAILS),
         (errors.IncompatibleHardwareMethodError(DETAILS), SAME_DETAILS),
     ]
     for (obj, check_details) in cases:
         self._test_class(obj, check_details)
Example #22
0
def _download_with_proxy(image_info, url, image_id):
    """Opens a download stream for the given URL.

    :param image_info: Image information dictionary.
    :param url: The URL string to request the image from.
    :param image_id: Image ID or URL for logging.

    :raises: ImageDownloadError if the download stream was not started
             properly.
    """
    no_proxy = image_info.get('no_proxy')
    if no_proxy:
        os.environ['no_proxy'] = no_proxy
    proxies = image_info.get('proxies', {})
    verify, cert = utils.get_ssl_client_options(CONF)
    resp = None
    for attempt in range(CONF.image_download_connection_retries + 1):
        try:
            resp = requests.get(url, stream=True, proxies=proxies,
                                verify=verify, cert=cert,
                                timeout=CONF.image_download_connection_timeout)
            if resp.status_code != 200:
                msg = ('Received status code {} from {}, expected 200. '
                       'Response body: {}').format(resp.status_code, url,
                                                   resp.text)
                raise errors.ImageDownloadError(image_id, msg)
        except (errors.ImageDownloadError, requests.RequestException) as e:
            if (attempt == CONF.image_download_connection_retries
                    # NOTE(dtantsur): do not retry 4xx status codes
                    or (resp and resp.status_code < 500)):
                raise
            else:
                LOG.warning('Unable to connect to %s, retrying. Error: %s',
                            url, e)
                time.sleep(CONF.image_download_connection_retry_interval)
        else:
            break
    return resp
def _request_url(image_info, url):
    resp = requests.get(url, stream=True)
    if resp.status_code != 200:
        raise errors.ImageDownloadError(image_info['id'])
    return resp
Example #24
0
    def __init__(self, image_info, time_obj=None):
        """Initialize an instance of the ImageDownload class.

        Trys each URL in image_info successively until a URL returns a
        successful request code. Once the object is initialized, the user may
        retrieve chunks of the image through the standard python iterator
        interface until either the image is fully downloaded, or an error is
        encountered.

        :param image_info: Image information dictionary.
        :param time_obj: Optional time object to indicate when the image
                         download began. Defaults to None. If None, then
                         time.time() will be used to find the start time of
                         the download.

        :raises: ImageDownloadError if starting the image download fails for
                 any reason.
        """
        self._time = time_obj or time.time()
        self._image_info = image_info
        self._request = None

        # Determine the hash algorithm and value will be used for calculation
        # and verification, fallback to md5 if algorithm is not set or not
        # supported.
        algo = image_info.get('os_hash_algo')
        if algo and algo in hashlib.algorithms_available:
            self._hash_algo = hashlib.new(algo)
            self._expected_hash_value = image_info.get('os_hash_value')
        elif image_info.get('checksum'):
            try:
                self._hash_algo = hashlib.md5()
            except ValueError as e:
                message = ('Unable to proceed with image {} as the legacy '
                           'checksum indicator has been used, which makes use '
                           'the MD5 algorithm. This algorithm failed to load '
                           'due to the underlying operating system. Error: '
                           '{}').format(image_info['id'], str(e))
                LOG.error(message)
                raise errors.RESTError(details=message)
            self._expected_hash_value = image_info['checksum']
        else:
            message = ('Unable to verify image {} with available checksums. '
                       'Please make sure the specified \'os_hash_algo\' '
                       '(currently {}) is supported by this ramdisk, or '
                       'provide a md5 checksum via the \'checksum\' '
                       'field'.format(image_info['id'],
                                      image_info.get('os_hash_algo')))
            LOG.error(message)
            raise errors.RESTError(details=message)

        self._expected_hash_value = _fetch_checksum(self._expected_hash_value,
                                                    image_info)

        details = []
        for url in image_info['urls']:
            try:
                LOG.info("Attempting to download image from {}".format(url))
                self._request = _download_with_proxy(image_info, url,
                                                     image_info['id'])
            except errors.ImageDownloadError as e:
                failtime = time.time() - self._time
                log_msg = ('URL: {}; time: {} '
                           'seconds. Error: {}').format(
                               url, failtime, e.secondary_message)
                LOG.warning(log_msg)
                details.append(log_msg)
                continue
            else:
                break
        else:
            details = '\n '.join(details)
            raise errors.ImageDownloadError(image_info['id'], details)