def _download_image(image_info): starttime = time.time() resp = None for url in image_info['urls']: try: LOG.info("Attempting to download image from {0}".format(url)) resp = _request_url(image_info, url) except errors.ImageDownloadError as e: failtime = time.time() - starttime log_msg = ('Image download failed. URL: {0}; time: {1} seconds. ' 'Error: {2}') LOG.warning(log_msg.format(url, failtime, e.details)) continue else: break if resp is None: msg = 'Image download failed for all URLs.' raise errors.ImageDownloadError(image_info['id'], msg) image_location = _image_location(image_info) with open(image_location, 'wb') as f: try: for chunk in resp.iter_content(IMAGE_CHUNK_SIZE): f.write(chunk) except Exception as e: msg = 'Unable to write image to {0}. Error: {1}'.format( image_location, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) totaltime = time.time() - starttime LOG.info("Image downloaded from {0} in {1} seconds".format( image_location, totaltime)) if not _verify_image(image_info, image_location): raise errors.ImageChecksumError(image_info['id'])
def _fetch_checksum(checksum, image_info): """Fetch checksum from remote location, if needed.""" if not (checksum.startswith('http://') or checksum.startswith('https://')): # Not a remote checksum, return as it is. return checksum LOG.debug('Downloading checksums file from %s', checksum) resp = _download_with_proxy(image_info, checksum, checksum).text lines = [line.strip() for line in resp.split('\n') if line.strip()] if not lines: raise errors.ImageDownloadError(checksum, "Empty checksum file") elif len(lines) == 1: # Special case - checksums file with only the checksum itself if ' ' not in lines[0]: return lines[0] # FIXME(dtantsur): can we assume the same name for all images? expected_fname = os.path.basename( urlparse.urlparse(image_info['urls'][0]).path) for line in lines: checksum, fname = line.strip().split(None, 1) # The star symbol designates binary mode, which is the same as text # mode on GNU systems. if fname.strip().lstrip('*') == expected_fname: return checksum.strip() raise errors.ImageDownloadError( checksum, "Checksum file does not contain name %s" % expected_fname)
def _download_image(image_info): starttime = time.time() resp = None for url in image_info['urls']: try: LOG.info("Attempting to download image from {0}".format(url)) resp = _request_url(image_info, url) except errors.ImageDownloadError: failtime = time.time() - starttime log_msg = "Image download failed. URL: {0}; time: {1} seconds" LOG.warning(log_msg.format(url, failtime)) continue else: break if resp is None: raise errors.ImageDownloadError(image_info['id']) image_location = _image_location(image_info) with open(image_location, 'wb') as f: try: for chunk in resp.iter_content(1024 * 1024): f.write(chunk) except Exception: raise errors.ImageDownloadError(image_info['id']) totaltime = time.time() - starttime LOG.info("Image downloaded from {0} in {1} seconds".format( image_location, totaltime)) if not _verify_image(image_info, image_location): raise errors.ImageChecksumError(image_info['id'])
def _request_url(image_info, url): resp = requests.get(url, stream=True) if resp.status_code != 200: msg = ('Received status code {0} from {1}, expected 200. Response ' 'body: {2}').format(resp.status_code, url, resp.text) raise errors.ImageDownloadError(image_info['id'], msg) return resp
def _stream_raw_image_onto_device(self, image_info, device): """Streams raw image data to specified local device. :param image_info: Image information dictionary. :param device: The disk name, as a string, on which to store the image. Example: '/dev/sda' :raises: ImageDownloadError if the image download encounters an error. :raises: ImageChecksumError if the checksum of the local image does not match the checksum as reported by glance in image_info. """ starttime = time.time() image_download = ImageDownload(image_info, time_obj=starttime) with open(device, 'wb+') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = 'Unable to write image to device {0}. Error: {1}'.format( device, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) totaltime = time.time() - starttime LOG.info("Image streamed onto device {0} in {1} " "seconds".format(device, totaltime)) # Verify if the checksum of the streamed image is correct _verify_image(image_info, device, image_download.md5sum())
def _download_with_proxy(image_info, url, image_id): """Opens a download stream for the given URL. :param image_info: Image information dictionary. :param url: The URL string to request the image from. :param image_id: Image ID or URL for logging. :raises: ImageDownloadError if the download stream was not started properly. """ no_proxy = image_info.get('no_proxy') if no_proxy: os.environ['no_proxy'] = no_proxy proxies = image_info.get('proxies', {}) verify, cert = utils.get_ssl_client_options(CONF) resp = requests.get(url, stream=True, proxies=proxies, verify=verify, cert=cert) if resp.status_code != 200: msg = ('Received status code {} from {}, expected 200. Response ' 'body: {}').format(resp.status_code, url, resp.text) raise errors.ImageDownloadError(image_id, msg) return resp
def _download_image(image_info): """Downloads the specified image to the local file system. :param image_info: Image information dictionary. :raises: ImageDownloadError if the image download fails for any reason. :raises: ImageChecksumError if the downloaded image's checksum does not match the one reported in image_info. """ starttime = time.time() image_location = _image_location(image_info) image_download = ImageDownload(image_info, time_obj=starttime) with open(image_location, 'wb') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = 'Unable to write image to {0}. Error: {1}'.format( image_location, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) totaltime = time.time() - starttime LOG.info("Image downloaded from {0} in {1} seconds".format(image_location, totaltime)) _verify_image(image_info, image_location, image_download.md5sum())
def __iter__(self): """Downloads and returns the next chunk of the image. :returns: A chunk of the image. Size of chunk is IMAGE_CHUNK_SIZE which is a constant in this module. """ self._last_chunk_time = None for chunk in self._request.iter_content(IMAGE_CHUNK_SIZE): # Per requests forum posts/discussions, iter_content should # periodically yield to the caller for the client to do things # like stopwatch and potentially interrupt the download. # While this seems weird and doesn't exactly seem to match the # patterns in requests and urllib3, it does appear to be the # case. Field testing in environments where TCP sockets were # discovered in a read hanged state were navigated with # this code. if chunk: self._last_chunk_time = time.time() self._hash_algo.update(chunk) yield chunk elif (time.time() - self._last_chunk_time > CONF.image_download_connection_timeout): LOG.error('Timeout reached waiting for a chunk of data from ' 'a remote server.') raise errors.ImageDownloadError( self._image_info['id'], 'Timed out reading next chunk from webserver')
def _stream_raw_image_onto_device(self, image_info, device): """Streams raw image data to specified local device. :param image_info: Image information dictionary. :param device: The disk name, as a string, on which to store the image. Example: '/dev/sda' :raises: ImageDownloadError if the image download encounters an error. :raises: ImageChecksumError if the checksum of the local image does not match the checksum as reported by glance in image_info. """ starttime = time.time() image_download = ImageDownload(image_info, time_obj=starttime) with open(device, 'wb+') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = 'Unable to write image to device {}. Error: {}'.format( device, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) totaltime = time.time() - starttime LOG.info("Image streamed onto device {} in {} " "seconds".format(device, totaltime)) # Verify if the checksum of the streamed image is correct image_download.verify_image(device) # Fix any gpt partition try: disk_utils.fix_gpt_partition(device, node_uuid=None) except exception.InstanceDeployFailure: # Note: the catch internal to the helper method logs any errors. pass
def __init__(self, image_info, time_obj=None): """Initialize an instance of the ImageDownload class. Trys each URL in image_info successively until a URL returns a successful request code. Once the object is initialized, the user may retrieve chunks of the image through the standard python iterator interface until either the image is fully downloaded, or an error is encountered. :param image_info: Image information dictionary. :param time_obj: Optional time object to indicate when the image download began. Defaults to None. If None, then time.time() will be used to find the start time of the download. :raises: ImageDownloadError if starting the image download fails for any reason. """ self._time = time_obj or time.time() self._image_info = image_info self._request = None # Determine the hash algorithm and value will be used for calculation # and verification, fallback to md5 if algorithm is not set or not # supported. algo = image_info.get('os_hash_algo') if algo and algo in hashlib.algorithms_available: self._hash_algo = hashlib.new(algo) self._expected_hash_value = image_info.get('os_hash_value') else: self._hash_algo = hashlib.md5() self._expected_hash_value = image_info['checksum'] self._expected_hash_value = _fetch_checksum(self._expected_hash_value, image_info) details = [] for url in image_info['urls']: try: LOG.info("Attempting to download image from {}".format(url)) self._request = _download_with_proxy(image_info, url, image_info['id']) except errors.ImageDownloadError as e: failtime = time.time() - self._time log_msg = ('URL: {}; time: {} ' 'seconds. Error: {}').format( url, failtime, e.secondary_message) LOG.warning(log_msg) details.append(log_msg) continue else: break else: details = '\n '.join(details) raise errors.ImageDownloadError(image_info['id'], details)
def _download_file(self, image_info, url): no_proxy = image_info.get('no_proxy') if no_proxy: os.environ['no_proxy'] = no_proxy proxies = image_info.get('proxies', {}) resp = requests.get(url, stream=True, proxies=proxies) if resp.status_code != 200: msg = ('Received status code {0} from {1}, expected 200. Response ' 'body: {2}').format(resp.status_code, url, resp.text) raise errors.ImageDownloadError(image_info['id'], msg) return resp
def _download_with_proxy(image_info, url, image_id): """Opens a download stream for the given URL. :param image_info: Image information dictionary. :param url: The URL string to request the image from. :param image_id: Image ID or URL for logging. :raises: ImageDownloadError if the download stream was not started properly. """ no_proxy = image_info.get('no_proxy') if no_proxy: os.environ['no_proxy'] = no_proxy proxies = image_info.get('proxies', {}) verify, cert = utils.get_ssl_client_options(CONF) resp = None for attempt in range(CONF.image_download_connection_retries + 1): try: # NOTE(TheJulia) The get request below does the following: # * Performs dns lookups, if necessary # * Opens the TCP socket to the remote host # * Negotiates TLS, if applicable # * Checks cert validity, if necessary, which may be # more tcp socket connections. # * Issues the get request and then returns back to the caller the # handler which is used to stream the data into the agent. # While this all may be at risk of transitory interrupts, most of # these socket will have timeouts applied to them, although not # exactly just as the timeout value exists. The risk in transitory # failure is more so once we've started the download and we are # processing the incoming data. resp = requests.get(url, stream=True, proxies=proxies, verify=verify, cert=cert, timeout=CONF.image_download_connection_timeout) if resp.status_code != 200: msg = ('Received status code {} from {}, expected 200. ' 'Response body: {}').format(resp.status_code, url, resp.text) raise errors.ImageDownloadError(image_id, msg) except (errors.ImageDownloadError, requests.RequestException) as e: if (attempt == CONF.image_download_connection_retries # NOTE(dtantsur): do not retry 4xx status codes or (resp and resp.status_code < 500)): raise else: LOG.warning('Unable to connect to %s, retrying. Error: %s', url, e) time.sleep(CONF.image_download_connection_retry_interval) else: break return resp
def _stream_raw_image_onto_device(self, image_info, device): """Streams raw image data to specified local device. :param image_info: Image information dictionary. :param device: The disk name, as a string, on which to store the image. Example: '/dev/sda' :raises: ImageDownloadError if the image download encounters an error. :raises: ImageChecksumError if the checksum of the local image does not match the checksum as reported by glance in image_info. """ starttime = time.time() total_retries = CONF.image_download_connection_retries for attempt in range(total_retries + 1): try: image_download = ImageDownload(image_info, time_obj=starttime) with open(device, 'wb+') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = ('Unable to write image to device {}. ' 'Error: {}').format(device, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) except errors.ImageDownloadError as e: if attempt == CONF.image_download_connection_retries: raise else: LOG.warning('Image download failed, %(attempt)s of ' '%(total)s: %(error)s', {'attempt': attempt, 'total': total_retries, 'error': e}) time.sleep(CONF.image_download_connection_retry_interval) else: break totaltime = time.time() - starttime LOG.info("Image streamed onto device {} in {} " "seconds".format(device, totaltime)) # Verify if the checksum of the streamed image is correct image_download.verify_image(device) # Fix any gpt partition try: disk_utils.fix_gpt_partition(device, node_uuid=None) except exception.InstanceDeployFailure: # Note: the catch internal to the helper method logs any errors. pass # Fix the root partition UUID root_uuid = disk_utils.block_uuid(device) LOG.info("{} UUID is now {}".format(device, root_uuid)) self.partition_uuids['root uuid'] = root_uuid
def _stream_raw_image_onto_device(self, image_info, device): """Streams raw image data to specified local device. :param image_info: Image information dictionary. :param device: The disk name, as a string, on which to store the image. Example: '/dev/sda' :raises: ImageDownloadError if the image download encounters an error. :raises: ImageChecksumError if the checksum of the local image does not match the checksum as reported by glance in image_info. """ starttime = time.time() total_retries = CONF.image_download_connection_retries for attempt in range(total_retries + 1): try: image_download = ImageDownload(image_info, time_obj=starttime) with open(device, 'wb+') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = ('Unable to write image to device {}. ' 'Error: {}').format(device, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) except errors.ImageDownloadError as e: if attempt == CONF.image_download_connection_retries: raise else: LOG.warning( 'Image download failed, %(attempt)s of ' '%(total)s: %(error)s', { 'attempt': attempt, 'total': total_retries, 'error': e }) time.sleep(CONF.image_download_connection_retry_interval) else: break totaltime = time.time() - starttime LOG.info("Image streamed onto device {} in {} " "seconds".format(device, totaltime)) # Verify if the checksum of the streamed image is correct image_download.verify_image(device)
def _download_image(image_info): starttime = time.time() image_location = _image_location(image_info) image_download = ImageDownload(image_info, time_obj=starttime) with open(image_location, 'wb') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = 'Unable to write image to {0}. Error: {1}'.format( image_location, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) totaltime = time.time() - starttime LOG.info("Image downloaded from {0} in {1} seconds".format( image_location, totaltime)) _verify_image(image_info, image_location, image_download.md5sum())
def _stream_raw_image_onto_device(self, image_info, device): starttime = time.time() image_download = ImageDownload(image_info, time_obj=starttime) with open(device, 'wb+') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = 'Unable to write image to device {0}. Error: {1}'.format( device, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) totaltime = time.time() - starttime LOG.info("Image streamed onto device {0} in {1} " "seconds".format(device, totaltime)) # Verify if the checksum of the streamed image is correct _verify_image(image_info, device, image_download.md5sum())
def _download_file(self, image_info, url): """Opens a download stream for the given URL. :param image_info: Image information dictionary. :param url: The URL string to request the image from. :raises: ImageDownloadError if the download stream was not started properly. """ no_proxy = image_info.get('no_proxy') if no_proxy: os.environ['no_proxy'] = no_proxy proxies = image_info.get('proxies', {}) resp = requests.get(url, stream=True, proxies=proxies) if resp.status_code != 200: msg = ('Received status code {0} from {1}, expected 200. Response ' 'body: {2}').format(resp.status_code, url, resp.text) raise errors.ImageDownloadError(image_info['id'], msg) return resp
def __init__(self, image_info, time_obj=None): self._md5checksum = hashlib.md5() self._time = time_obj or time.time() self._request = None for url in image_info['urls']: try: LOG.info("Attempting to download image from {0}".format(url)) self._request = self._download_file(image_info, url) except errors.ImageDownloadError as e: failtime = time.time() - self._time log_msg = ('Image download failed. URL: {0}; time: {1} ' 'seconds. Error: {2}') LOG.warning(log_msg.format(url, failtime, e.details)) continue else: break else: msg = 'Image download failed for all URLs.' raise errors.ImageDownloadError(image_info['id'], msg)
def __init__(self, image_info, time_obj=None): """Initialize an instance of the ImageDownload class. Trys each URL in image_info successively until a URL returns a successful request code. Once the object is initialized, the user may retrieve chunks of the image through the standard python iterator interface until either the image is fully downloaded, or an error is encountered. :param image_info: Image information dictionary. :param time_obj: Optional time object to indicate when the image download began. Defaults to None. If None, then time.time() will be used to find the start time of the download. :raises: ImageDownloadError if starting the image download fails for any reason. """ self._md5checksum = hashlib.md5() self._time = time_obj or time.time() self._request = None details = [] for url in image_info['urls']: try: LOG.info("Attempting to download image from {}".format(url)) self._request = self._download_file(image_info, url) except errors.ImageDownloadError as e: failtime = time.time() - self._time log_msg = ('URL: {}; time: {} ' 'seconds. Error: {}').format( url, failtime, e.details) LOG.warning('Image download failed. %s', log_msg) details += log_msg continue else: break else: details = '/n'.join(details) msg = ('Image download failed for all URLs with following errors: ' '{}'.format(details)) raise errors.ImageDownloadError(image_info['id'], msg)
def _download_image(image_info): """Downloads the specified image to the local file system. :param image_info: Image information dictionary. :raises: ImageDownloadError if the image download fails for any reason. :raises: ImageChecksumError if the downloaded image's checksum does not match the one reported in image_info. """ starttime = time.time() image_location = _image_location(image_info) for attempt in range(CONF.image_download_connection_retries + 1): try: image_download = ImageDownload(image_info, time_obj=starttime) with open(image_location, 'wb') as f: try: for chunk in image_download: f.write(chunk) except Exception as e: msg = 'Unable to write image to {}. Error: {}'.format( image_location, str(e)) raise errors.ImageDownloadError(image_info['id'], msg) except errors.ImageDownloadError as e: if attempt == CONF.image_download_connection_retries: raise else: LOG.warning( 'Image download failed, %(attempt)s of %(total)s: ' '%(error)s', { 'attempt': attempt, 'total': CONF.image_download_connection_retries, 'error': e }) time.sleep(CONF.image_download_connection_retry_interval) else: break totaltime = time.time() - starttime LOG.info("Image downloaded from {} in {} seconds".format( image_location, totaltime)) image_download.verify_image(image_location)
def test_error_classes(self): cases = [ (errors.InvalidContentError(DETAILS), SAME_DETAILS), (errors.NotFound(), SAME_CL_DETAILS), (errors.CommandExecutionError(DETAILS), SAME_DETAILS), (errors.InvalidCommandError(DETAILS), SAME_DETAILS), (errors.InvalidCommandParamsError(DETAILS), SAME_DETAILS), (errors.RequestedObjectNotFoundError('type_descr', 'obj_id'), DIFF_CL_DETAILS), (errors.IronicAPIError(DETAILS), SAME_DETAILS), (errors.HeartbeatError(DETAILS), SAME_DETAILS), (errors.LookupNodeError(DETAILS), SAME_DETAILS), (errors.LookupAgentIPError(DETAILS), SAME_DETAILS), (errors.LookupAgentInterfaceError(DETAILS), SAME_DETAILS), (errors.ImageDownloadError('image_id', DETAILS), DIFF_CL_DETAILS), (errors.ImageChecksumError('image_id', '/foo/image_id', 'incorrect', 'correct'), DIFF_CL_DETAILS), (errors.ImageWriteError('device', 'exit_code', 'stdout', 'stderr'), DIFF_CL_DETAILS), (errors.ConfigDriveTooLargeError('filename', 'filesize'), DIFF_CL_DETAILS), (errors.ConfigDriveWriteError('device', 'exit_code', 'stdout', 'stderr'), DIFF_CL_DETAILS), (errors.SystemRebootError('exit_code', 'stdout', 'stderr'), DIFF_CL_DETAILS), (errors.BlockDeviceEraseError(DETAILS), SAME_DETAILS), (errors.BlockDeviceError(DETAILS), SAME_DETAILS), (errors.VirtualMediaBootError(DETAILS), SAME_DETAILS), (errors.UnknownNodeError(), DEFAULT_DETAILS), (errors.UnknownNodeError(DETAILS), SAME_DETAILS), (errors.HardwareManagerNotFound(), DEFAULT_DETAILS), (errors.HardwareManagerNotFound(DETAILS), SAME_DETAILS), (errors.HardwareManagerMethodNotFound('method'), DIFF_CL_DETAILS), (errors.IncompatibleHardwareMethodError(), DEFAULT_DETAILS), (errors.IncompatibleHardwareMethodError(DETAILS), SAME_DETAILS), ] for (obj, check_details) in cases: self._test_class(obj, check_details)
def _download_with_proxy(image_info, url, image_id): """Opens a download stream for the given URL. :param image_info: Image information dictionary. :param url: The URL string to request the image from. :param image_id: Image ID or URL for logging. :raises: ImageDownloadError if the download stream was not started properly. """ no_proxy = image_info.get('no_proxy') if no_proxy: os.environ['no_proxy'] = no_proxy proxies = image_info.get('proxies', {}) verify, cert = utils.get_ssl_client_options(CONF) resp = None for attempt in range(CONF.image_download_connection_retries + 1): try: resp = requests.get(url, stream=True, proxies=proxies, verify=verify, cert=cert, timeout=CONF.image_download_connection_timeout) if resp.status_code != 200: msg = ('Received status code {} from {}, expected 200. ' 'Response body: {}').format(resp.status_code, url, resp.text) raise errors.ImageDownloadError(image_id, msg) except (errors.ImageDownloadError, requests.RequestException) as e: if (attempt == CONF.image_download_connection_retries # NOTE(dtantsur): do not retry 4xx status codes or (resp and resp.status_code < 500)): raise else: LOG.warning('Unable to connect to %s, retrying. Error: %s', url, e) time.sleep(CONF.image_download_connection_retry_interval) else: break return resp
def _request_url(image_info, url): resp = requests.get(url, stream=True) if resp.status_code != 200: raise errors.ImageDownloadError(image_info['id']) return resp
def __init__(self, image_info, time_obj=None): """Initialize an instance of the ImageDownload class. Trys each URL in image_info successively until a URL returns a successful request code. Once the object is initialized, the user may retrieve chunks of the image through the standard python iterator interface until either the image is fully downloaded, or an error is encountered. :param image_info: Image information dictionary. :param time_obj: Optional time object to indicate when the image download began. Defaults to None. If None, then time.time() will be used to find the start time of the download. :raises: ImageDownloadError if starting the image download fails for any reason. """ self._time = time_obj or time.time() self._image_info = image_info self._request = None # Determine the hash algorithm and value will be used for calculation # and verification, fallback to md5 if algorithm is not set or not # supported. algo = image_info.get('os_hash_algo') if algo and algo in hashlib.algorithms_available: self._hash_algo = hashlib.new(algo) self._expected_hash_value = image_info.get('os_hash_value') elif image_info.get('checksum'): try: self._hash_algo = hashlib.md5() except ValueError as e: message = ('Unable to proceed with image {} as the legacy ' 'checksum indicator has been used, which makes use ' 'the MD5 algorithm. This algorithm failed to load ' 'due to the underlying operating system. Error: ' '{}').format(image_info['id'], str(e)) LOG.error(message) raise errors.RESTError(details=message) self._expected_hash_value = image_info['checksum'] else: message = ('Unable to verify image {} with available checksums. ' 'Please make sure the specified \'os_hash_algo\' ' '(currently {}) is supported by this ramdisk, or ' 'provide a md5 checksum via the \'checksum\' ' 'field'.format(image_info['id'], image_info.get('os_hash_algo'))) LOG.error(message) raise errors.RESTError(details=message) self._expected_hash_value = _fetch_checksum(self._expected_hash_value, image_info) details = [] for url in image_info['urls']: try: LOG.info("Attempting to download image from {}".format(url)) self._request = _download_with_proxy(image_info, url, image_info['id']) except errors.ImageDownloadError as e: failtime = time.time() - self._time log_msg = ('URL: {}; time: {} ' 'seconds. Error: {}').format( url, failtime, e.secondary_message) LOG.warning(log_msg) details.append(log_msg) continue else: break else: details = '\n '.join(details) raise errors.ImageDownloadError(image_info['id'], details)