def upload_volume(context, image_service, image_meta, volume_path):
image_id = image_meta['id']
if (image_meta['disk_format'] == 'raw'):
LOG.debug("%s was raw, no need to convert to %s" %
(image_id, image_meta['disk_format']))
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir
and not os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was raw, converting to %s" %
(image_id, image_meta['disk_format']))
convert_image(volume_path, tmp, image_meta['disk_format'])
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
os.unlink(tmp)
def test_backup_volume(self):
self.mox.StubOutWithMock(self._driver, 'db')
self.mox.StubOutWithMock(self._driver.db, 'volume_get')
volume = {'id': '2', 'name': self.TEST_VOLNAME}
self._driver.db.volume_get(context, volume['id']).AndReturn(volume)
info = imageutils.QemuImgInfo()
info.file_format = 'raw'
self.mox.StubOutWithMock(image_utils, 'qemu_img_info')
image_utils.qemu_img_info(self.TEST_VOLPATH).AndReturn(info)
self.mox.StubOutWithMock(utils, 'temporary_chown')
mock_tempchown = self.mox.CreateMockAnything()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
mock_tempchown.__enter__()
mock_tempchown.__exit__(None, None, None)
self.mox.StubOutWithMock(fileutils, 'file_open')
mock_fileopen = self.mox.CreateMockAnything()
fileutils.file_open(self.TEST_VOLPATH).AndReturn(mock_fileopen)
mock_fileopen.__enter__()
mock_fileopen.__exit__(None, None, None)
backup = {'volume_id': volume['id']}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.backup(backup, mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.backup_volume(context, backup, mock_servicebackup)
self.mox.VerifyAll()
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw', run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s",
image_id, volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s",
image_id, volume_format, image_meta['disk_format'])
convert_image(volume_path, tmp, image_meta['disk_format'],
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def upload_volume(context, image_service, image_meta, volume_path):
image_id = image_meta['id']
if (image_meta['disk_format'] == 'raw'):
LOG.debug("%s was raw, no need to convert to %s" %
(image_id, image_meta['disk_format']))
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was raw, converting to %s" %
(image_id, image_meta['disk_format']))
convert_image(volume_path, tmp, image_meta['disk_format'])
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
os.unlink(tmp)
def upload_volume(context, image_service, image_meta, volume_path, volume_format="raw"):
image_id = image_meta["id"]
if image_meta["disk_format"] == volume_format:
LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta["disk_format"]))
if os.name == "nt":
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta["disk_format"]))
convert_image(volume_path, tmp, image_meta["disk_format"])
data = qemu_img_info(tmp)
if data.file_format != image_meta["disk_format"]:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s")
% {"f1": image_meta["disk_format"], "f2": data.file_format},
)
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def test_backup_volume(self):
self.mox.StubOutWithMock(self._driver, "db")
self.mox.StubOutWithMock(self._driver.db, "volume_get")
volume = {"id": "2", "name": self.TEST_VOLNAME}
self._driver.db.volume_get(context, volume["id"]).AndReturn(volume)
info = imageutils.QemuImgInfo()
info.file_format = "raw"
self.mox.StubOutWithMock(image_utils, "qemu_img_info")
image_utils.qemu_img_info(self.TEST_VOLPATH).AndReturn(info)
self.mox.StubOutWithMock(utils, "temporary_chown")
mock_tempchown = self.mox.CreateMockAnything()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
mock_tempchown.__enter__()
mock_tempchown.__exit__(None, None, None)
self.mox.StubOutWithMock(fileutils, "file_open")
mock_fileopen = self.mox.CreateMockAnything()
fileutils.file_open(self.TEST_VOLPATH).AndReturn(mock_fileopen)
mock_fileopen.__enter__()
mock_fileopen.__exit__(None, None, None)
backup = {"volume_id": volume["id"]}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.backup(backup, mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.backup_volume(context, backup, mock_servicebackup)
self.mox.VerifyAll()
def _backup_device(self, context, backup, backup_service, device):
"""Create a new backup from a volume."""
LOG.debug('Creating a new backup for %s.', device['name'])
use_multipath = self.configuration.use_multipath_for_image_xfer
enforce_multipath = self.configuration.enforce_multipath_for_image_xfer
properties = utils.brick_get_connector_properties(
use_multipath, enforce_multipath)
attach_info, device = self._attach_volume(context, device, properties)
try:
volume_path = attach_info['device']['path']
# Secure network file systems will not chown files.
if self.secure_file_operations_enabled():
with fileutils.file_open(volume_path) as volume_file:
backup_info = backup_service.backup(backup, volume_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_info = backup_service.backup(
backup, volume_file)
finally:
self._detach_volume(context, attach_info, device, properties)
return backup_info
def test_backup_volume(self):
self.mox = mox_lib.Mox()
self._driver.db = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(self._driver.db, 'volume_get')
volume = {'id': '2', 'name': self.TEST_VOLNAME}
self._driver.db.volume_get(context, volume['id']).AndReturn(volume)
info = imageutils.QemuImgInfo()
info.file_format = 'raw'
self.mox.StubOutWithMock(image_utils, 'qemu_img_info')
image_utils.qemu_img_info(self.TEST_VOLPATH).AndReturn(info)
self.mox.StubOutWithMock(utils, 'temporary_chown')
mock_tempchown = mock.MagicMock()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
self.mox.StubOutWithMock(fileutils, 'file_open')
mock_fileopen = mock.MagicMock()
fileutils.file_open(self.TEST_VOLPATH).AndReturn(mock_fileopen)
backup = {'volume_id': volume['id']}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.backup(backup, mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.backup_volume(context, backup, mock_servicebackup)
def upload_volume(context,
image_service,
image_meta,
volume_path,
volume_format='raw',
run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s", image_id,
volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s", image_id, volume_format,
image_meta['disk_format'])
data = qemu_img_info(volume_path, run_as_root=run_as_root)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {
'fmt': fmt,
'backing_file': backing_file
})
convert_image(volume_path,
tmp,
image_meta['disk_format'],
data.virtual_size,
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw'):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
data = qemu_img_info(volume_path)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file})
convert_image(volume_path, tmp, image_meta['disk_format'],
bps_limit=CONF.volume_copy_bps_limit)
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def upload_volume_to_vgw(
context, image_service, image_meta, volume_path, volume, vgw_url, volume_format="raw", run_as_root=True
):
LOG.error("begin time of upload_volume_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
image_id = image_meta["id"]
volume_id = volume["id"]
if image_meta["disk_format"] == volume_format:
LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta["disk_format"]))
if os.name == "nt" or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, "rb") as files:
r = requests.post(vgw_url, data=files)
if r.status_code != 200:
# LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(reason=("upload the volume %s back_file failed" % volume_id))
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as files:
r = requests.post(vgw_url, data=files)
# LOG.debug('the request result is %s' %(str(r.status_code)))
if r.status_code != 200:
# LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(reason=("upload the volume %s back_file failed" % volume_id))
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta["disk_format"]))
LOG.error("begin time of convert_image is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
convert_image(volume_path, tmp, image_meta["disk_format"])
LOG.error("end time of upload_volume_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
data = qemu_img_info(tmp)
if data.file_format != image_meta["disk_format"]:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s")
% {"f1": image_meta["disk_format"], "f2": data.file_format},
)
with fileutils.file_open(tmp, "rb") as files:
LOG.error("begin time of upload file is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
r = requests.post(vgw_url, data=files)
# LOG.debug('the request result is %' %(str(r.status_code)))
if r.status_code != 200:
# LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(reason=("upload the volume %s back_file failed" % volume_id))
LOG.error("end time of upload file is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
# todo delete the tmp file
fileutils.delete_if_exists(tmp)
LOG.error("end time of upload_volume_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_("FuseVolumeDriver::restore_backup() volume=%s") % volume)
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_('Begin restore of backup %s.') % backup['id'])
volume_path = self.local_path(volume)
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_('Begin restore of backup %s.') % backup['id'])
volume_path = self.local_path(volume)
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
def put_object(self, container, name, reader, content_length=None,
etag=None, chunk_size=None, content_type=None,
headers=None, query_string=None):
object_path = tempfile.gettempdir() + '/' + container + '/' + name
with fileutils.file_open(object_path, 'wb') as object_file:
object_file.write(reader.read())
return hashlib.md5(reader.read()).hexdigest()
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.info(_LI('Restoring backup %(backup)s to volume %(volume)s.') %
{'backup': backup['id'], 'volume': volume['name']})
volume_local_path = self.local_path(volume)
with utils.temporary_chown(volume_local_path):
with fileutils.file_open(volume_local_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def upload_volume(context,
image_service,
image_meta,
volume_path,
volume_format='raw',
run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir
and not os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
convert_image(volume_path,
tmp,
image_meta['disk_format'],
bps_limit=CONF.volume_copy_bps_limit,
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug("Begin restore of backup %s." % backup["id"])
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, "wb") as volume_file:
backup_service.restore(backup, volume["id"], volume_file)
def get_object(self, container, name):
LOG.debug("fake get_object %(container)s, %(name)s.",
{'container': container,
'name': name})
if container == 'socket_error_on_get':
raise socket.error(111, 'ECONNREFUSED')
object_path = tempfile.gettempdir() + '/' + container + '/' + name
with fileutils.file_open(object_path, 'rb') as object_file:
return (None, object_file.read())
def _download_vmdk_from_vcloud(self, context, src_url, dst_file_name):
local_file_handle = fileutils.file_open(dst_file_name, "wb")
remote_file_handle = urllib2.urlopen(src_url)
file_size = remote_file_handle.headers['content-length']
util.start_transfer(context,
IMAGE_TRANSFER_TIMEOUT_SECS,
remote_file_handle,
file_size,
write_file_handle=local_file_handle)
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw', run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s",
image_id, volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s",
image_id, volume_format, image_meta['disk_format'])
data = qemu_img_info(volume_path, run_as_root=run_as_root)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file})
convert_image(volume_path, tmp, image_meta['disk_format'],
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def test_restore_backup(self):
volume = {'id': '2', 'name': self.TEST_VOLNAME}
self.mox.StubOutWithMock(utils, 'temporary_chown')
mock_tempchown = mock.MagicMock()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
self.mox.StubOutWithMock(fileutils, 'file_open')
mock_fileopen = mock.MagicMock()
fileutils.file_open(self.TEST_VOLPATH, 'wb').AndReturn(mock_fileopen)
backup = {'id': 123, 'volume_id': volume['id']}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.restore(backup, volume['id'], mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.restore_backup(context, backup, volume,
mock_servicebackup)
def test_restore_backup(self):
volume = {'id': '2', 'name': self.TEST_VOLNAME}
self.mox.StubOutWithMock(utils, 'temporary_chown')
mock_tempchown = mock.MagicMock()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
self.mox.StubOutWithMock(fileutils, 'file_open')
mock_fileopen = mock.MagicMock()
fileutils.file_open(self.TEST_VOLPATH, 'wb').AndReturn(mock_fileopen)
backup = {'id': 123, 'volume_id': volume['id']}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.restore(backup, volume['id'], mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.restore_backup(context, backup, volume,
mock_servicebackup)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.info(
_LI("Restoring backup %(backup)s to volume %(volume)s.")
% {"backup": backup["id"], "volume": volume["name"]}
)
volume_local_path = self.local_path(volume)
with utils.temporary_chown(volume_local_path):
with fileutils.file_open(volume_local_path, "wb") as volume_file:
backup_service.restore(backup, volume["id"], volume_file)
def get_object(self, container, name):
LOG.debug("fake get_object %(container)s, %(name)s.", {
'container': container,
'name': name
})
if container == 'socket_error_on_get':
raise socket.error(111, 'ECONNREFUSED')
object_path = tempfile.gettempdir() + '/' + container + '/' + name
with fileutils.file_open(object_path, 'rb') as object_file:
return (None, object_file.read())
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.info(_LI('Restoring backup %(backup)s to volume %(volume)s.'), {
'backup': backup['id'],
'volume': volume['name']
})
volume_local_path = self.local_path(volume)
with utils.temporary_chown(volume_local_path):
with fileutils.file_open(volume_local_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
image_id = image_meta["id"]
with image_utils.temporary_file() as tmp:
# image_utils.convert_image doesn't support "sheepdog:" source,
# so we use the qemu-img directly.
# Sheepdog volume is always raw-formatted.
cmd = ("qemu-img", "convert", "-f", "raw", "-t", "none", "-O", "raw", "sheepdog:%s" % volume["name"], tmp)
self._try_execute(*cmd)
with fileutils.file_open(tmp, "rb") as image_file:
image_service.update(context, image_id, {}, image_file)
def test_restore_backup(self):
volume = {"id": "2", "name": self.TEST_VOLNAME}
self.mox.StubOutWithMock(utils, "temporary_chown")
mock_tempchown = self.mox.CreateMockAnything()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
mock_tempchown.__enter__()
mock_tempchown.__exit__(None, None, None)
self.mox.StubOutWithMock(fileutils, "file_open")
mock_fileopen = self.mox.CreateMockAnything()
fileutils.file_open(self.TEST_VOLPATH, "wb").AndReturn(mock_fileopen)
mock_fileopen.__enter__()
mock_fileopen.__exit__(None, None, None)
backup = {"id": 123, "volume_id": volume["id"]}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.restore(backup, volume["id"], mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.restore_backup(context, backup, volume, mock_servicebackup)
self.mox.VerifyAll()
def upload_volume(context,
image_service,
image_meta,
volume_path,
volume_format='raw',
run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
convert_image(volume_path,
tmp,
image_meta['disk_format'],
bps_limit=CONF.volume_copy_bps_limit,
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
image_id = image_meta['id']
with image_utils.temporary_file() as tmp:
# image_utils.convert_image doesn't support "sheepdog:" source,
# so we use the qemu-img directly.
# Sheepdog volume is always raw-formatted.
cmd = ('qemu-img', 'convert', '-f', 'raw', '-t', 'none', '-O',
'raw', 'sheepdog:%s' % volume['name'], tmp)
self._try_execute(*cmd)
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw', run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
convert_image(volume_path, tmp, image_meta['disk_format'],
bps_limit=CONF.volume_copy_bps_limit,
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def put_object(self,
container,
name,
reader,
content_length=None,
etag=None,
chunk_size=None,
content_type=None,
headers=None,
query_string=None):
object_path = tempfile.gettempdir() + '/' + container + '/' + name
with fileutils.file_open(object_path, 'wb') as object_file:
object_file.write(reader.read())
return hashlib.md5(reader.read()).hexdigest()
def upload_volume(self, context, image_service, image_meta, volume_path):
LOG.debug("Uploading volume %s: " ,volume_path)
image_id = image_meta['id']
if (image_meta['disk_format'] == 'vhd'):
LOG.debug("%s was raw, no need to convert to %s" %
(image_id, image_meta['disk_format']))
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
#Copy the volume to the image conversion dir
temp_vhd_path = os.path.join(CONF.image_conversion_dir, str(image_meta['id']) + ".vhd")
self.copy_vhd_disk(volume_path, temp_vhd_path)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was vhd, converting to %s" %
(image_id, image_meta['disk_format']))
self.convert_image(temp_vhd_path, tmp, image_meta['disk_format'])
data = self.qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
LOG.debug("Converted size of %s is: %s", data.backing_file, data.disk_size)
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, image_meta, image_file)
os.unlink(tmp)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_('Restoring backup %(backup)s to '
'volume %(volume)s.') %
{'backup': backup['id'],
'volume': volume['name']})
(properties,attach_info) = self._attach_volume_syc(context, volume)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
finally:
self._detach_and_terminate_syc(context,attach_info,volume,properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup["volume_id"])
LOG.debug(("Creating a new backup for volume %s.") % volume["name"])
properties = utils.brick_get_connector_properties()
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info["device"]["path"]
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
self._detach_volume(context, attach_info, volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
LOG.debug(_('Creating a new backup for volume %s.') % volume['name'])
properties = utils.brick_get_connector_properties()
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
self._detach_volume(context, attach_info, volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
LOG.debug(_('Creating a new backup for volume %s.') %
volume['name'])
(properties,attach_info) = self._attach_volume_syc(context, volume)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
self._detach_and_terminate_syc(context,attach_info,volume,properties)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(
("Restoring backup %(backup)s to " "volume %(volume)s.")
% {"backup": backup["id"], "volume": volume["name"]}
)
properties = utils.brick_get_connector_properties()
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info["device"]["path"]
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, "wb") as volume_file:
backup_service.restore(backup, volume["id"], volume_file)
finally:
self._detach_volume(context, attach_info, volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup["volume_id"])
volume_local_path = self.local_path(volume)
LOG.info(_LI("Begin backup of volume %s.") % volume["name"])
qemu_img_info = image_utils.qemu_img_info(volume_local_path)
if qemu_img_info.file_format != "raw":
msg = _("Backup is only supported for raw-formatted " "SOFS volumes.")
raise exception.InvalidVolume(msg)
if qemu_img_info.backing_file is not None:
msg = _("Backup is only supported for SOFS volumes " "without backing file.")
raise exception.InvalidVolume(msg)
with utils.temporary_chown(volume_local_path):
with fileutils.file_open(volume_local_path) as volume_file:
backup_service.backup(backup, volume_file)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_('Restoring backup %(backup)s to '
'volume %(volume)s.') %
{'backup': backup['id'],
'volume': volume['name']})
properties = utils.brick_get_connector_properties()
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
finally:
self._detach_volume(context, attach_info, volume, properties)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
image_id = image_meta['id']
with image_utils.temporary_file() as tmp:
# image_utils.convert_image doesn't support "sheepdog:" source,
# so we use the qemu-img directly.
# Sheepdog volume is always raw-formatted.
cmd = ('qemu-img',
'convert',
'-f', 'raw',
'-t', 'none',
'-O', 'raw',
'sheepdog:%s' % volume['name'],
tmp)
self._try_execute(*cmd)
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
LOG.debug(_('Creating a new backup for volume %s.') % volume['name'])
root_helper = 'sudo cinder-rootwrap %s' % CONF.rootwrap_config
properties = initiator.get_connector_properties(root_helper)
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
self._detach_volume(attach_info)
self.terminate_connection(volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
volume_path = self.local_path(volume)
LOG.debug(_('Begin backup of volume %s.') % volume['name'])
# create a snapshot that will be used as the backup source
backup_path = '%s_%s.snap' % (volume_path, backup['id'])
self._create_gpfs_snap(volume_path, backup_path)
self._gpfs_redirect(volume_path)
try:
with fileutils.file_open(backup_path) as backup_file:
backup_service.backup(backup, backup_file)
finally:
# clean up snapshot file. If it is a clone parent, delete
# will fail silently, but be cleaned up when volume is
# eventually removed. This ensures we do not accumulate
# more than gpfs_max_clone_depth snap files.
self._delete_gpfs_file(backup_path)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
volume_path = self.local_path(volume)
LOG.debug(_('Begin backup of volume %s.') % volume['name'])
# create a snapshot that will be used as the backup source
backup_path = '%s_%s.snap' % (volume_path, backup['id'])
self._create_gpfs_snap(volume_path, backup_path)
self._gpfs_redirect(volume_path)
try:
with fileutils.file_open(backup_path) as backup_file:
backup_service.backup(backup, backup_file)
finally:
# clean up snapshot file. If it is a clone parent, delete
# will fail silently, but be cleaned up when volume is
# eventually removed. This ensures we do not accumulate
# more than gpfs_max_clone_depth snap files.
self._delete_gpfs_file(backup_path)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_('Restoring backup %(backup)s to '
'volume %(volume)s.') %
{'backup': backup['id'],
'volume': volume['name']})
root_helper = 'sudo cinder-rootwrap %s' % CONF.rootwrap_config
properties = initiator.get_connector_properties(root_helper)
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
finally:
self._detach_volume(attach_info)
self.terminate_connection(volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
LOG.debug(_('Creating a new backup for volume %s.') %
volume['name'])
root_helper = 'sudo cinder-rootwrap %s' % CONF.rootwrap_config
properties = initiator.get_connector_properties(root_helper)
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
self._detach_volume(attach_info)
self.terminate_connection(volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
volume_local_path = self.local_path(volume)
LOG.info(_LI('Begin backup of volume %s.') % volume['name'])
qemu_img_info = image_utils.qemu_img_info(volume_local_path)
if qemu_img_info.file_format != 'raw':
msg = _('Backup is only supported for raw-formatted '
'SOFS volumes.')
raise exception.InvalidVolume(msg)
if qemu_img_info.backing_file is not None:
msg = _('Backup is only supported for SOFS volumes '
'without backing file.')
raise exception.InvalidVolume(msg)
with utils.temporary_chown(volume_local_path):
with fileutils.file_open(volume_local_path) as volume_file:
backup_service.backup(backup, volume_file)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug(_('Restoring backup %(backup)s to '
'volume %(volume)s.') %
{'backup': backup['id'],
'volume': volume['name']})
root_helper = 'sudo cinder-rootwrap %s' % CONF.rootwrap_config
properties = initiator.get_connector_properties(root_helper)
attach_info = self._attach_volume(context, volume, properties)
try:
volume_path = attach_info['device']['path']
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
finally:
self._detach_volume(attach_info)
self.terminate_connection(volume, properties)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup.volume_id)
temp_snapshot = None
previous_status = volume['previous_status']
if previous_status == 'in-use':
temp_snapshot = self._create_temp_snapshot(context, volume)
backup.temp_snapshot_id = temp_snapshot.id
backup.save()
volume_path = self.local_path(temp_snapshot)
else:
volume_path = self.local_path(volume)
try:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
if temp_snapshot:
self._delete_snapshot(context, temp_snapshot)
backup.temp_snapshot_id = None
backup.save()
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup.volume_id)
temp_snapshot = None
previous_status = volume['previous_status']
if previous_status == 'in-use':
temp_snapshot = self._create_temp_snapshot(context, volume)
backup.temp_snapshot_id = temp_snapshot.id
backup.save()
volume_path = self.local_path(temp_snapshot)
else:
volume_path = self.local_path(volume)
try:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
if temp_snapshot:
self._delete_snapshot(context, temp_snapshot)
backup.temp_snapshot_id = None
backup.save()
def get_object(self, container, name):
if container == 'socket_error_on_get':
raise socket.error(111, 'ECONNREFUSED')
object_path = tempfile.gettempdir() + '/' + container + '/' + name
with fileutils.file_open(object_path, 'rb') as object_file:
return (None, object_file.read())
def upload_volume_to_vgw(context,
image_service,
image_meta,
volume_path,
volume,
vgw_url,
volume_format='raw',
run_as_root=True):
LOG.error('begin time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
image_id = image_meta['id']
volume_id = volume['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as files:
r = requests.post(vgw_url, data=files)
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as files:
r = requests.post(vgw_url, data=files)
#LOG.debug('the request result is %s' %(str(r.status_code)))
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
LOG.error('begin time of convert_image is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
convert_image(volume_path, tmp, image_meta['disk_format'])
LOG.error('end time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp, 'rb') as files:
LOG.error('begin time of upload file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
r = requests.post(vgw_url, data=files)
#LOG.debug('the request result is %' %(str(r.status_code)))
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
LOG.error('end time of upload file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
#todo delete the tmp file
fileutils.delete_if_exists(tmp)
LOG.error('end time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
