def test_backup_volume(self):
self.mox = mox_lib.Mox()
self._driver.db = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(self._driver.db, 'volume_get')
volume = {'id': '2', 'name': self.TEST_VOLNAME}
self._driver.db.volume_get(context, volume['id']).AndReturn(volume)
info = imageutils.QemuImgInfo()
info.file_format = 'raw'
self.mox.StubOutWithMock(image_utils, 'qemu_img_info')
image_utils.qemu_img_info(self.TEST_VOLPATH).AndReturn(info)
self.mox.StubOutWithMock(utils, 'temporary_chown')
mock_tempchown = mock.MagicMock()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
self.mox.StubOutWithMock(fileutils, 'file_open')
mock_fileopen = mock.MagicMock()
fileutils.file_open(self.TEST_VOLPATH).AndReturn(mock_fileopen)
backup = {'volume_id': volume['id']}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.backup(backup, mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.backup_volume(context, backup, mock_servicebackup)
def test_backup_volume(self):
self.mox.StubOutWithMock(self._driver, "db")
self.mox.StubOutWithMock(self._driver.db, "volume_get")
volume = {"id": "2", "name": self.TEST_VOLNAME}
self._driver.db.volume_get(context, volume["id"]).AndReturn(volume)
info = imageutils.QemuImgInfo()
info.file_format = "raw"
self.mox.StubOutWithMock(image_utils, "qemu_img_info")
image_utils.qemu_img_info(self.TEST_VOLPATH).AndReturn(info)
self.mox.StubOutWithMock(utils, "temporary_chown")
mock_tempchown = self.mox.CreateMockAnything()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
mock_tempchown.__enter__()
mock_tempchown.__exit__(None, None, None)
self.mox.StubOutWithMock(fileutils, "file_open")
mock_fileopen = self.mox.CreateMockAnything()
fileutils.file_open(self.TEST_VOLPATH).AndReturn(mock_fileopen)
mock_fileopen.__enter__()
mock_fileopen.__exit__(None, None, None)
backup = {"volume_id": volume["id"]}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.backup(backup, mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.backup_volume(context, backup, mock_servicebackup)
self.mox.VerifyAll()
def _get_target_chap_auth(self, context, name):
vol_id = name.split(':')[1]
if os.path.exists(self.iet_conf):
try:
with utils.temporary_chown(self.iet_conf):
with open(self.iet_conf, 'r') as f:
iet_conf_text = f.readlines()
except Exception:
# If we fail to handle config file, raise exception here to
# prevent unexpected behavior during subsequent operations.
LOG.exception(_LE("Failed to open config for %s."), vol_id)
raise
target_found = False
for line in iet_conf_text:
if target_found:
m = re.search('(\w+) (\w+) (\w+)', line)
if m:
return (m.group(2), m.group(3))
else:
LOG.debug("Failed to find CHAP auth from config "
"for %s", vol_id)
return None
elif name in line:
target_found = True
else:
# Missing config file is unxepected sisuation. But we will create
# new config file during create_iscsi_target(). Just we warn the
# operator here.
LOG.warning(_LW("Failed to find CHAP auth from config for "
"%(vol_id)s. Config file %(conf)s does not "
"exist."),
{'vol_id': vol_id, 'conf': self.iet_conf})
return None
def remove_iscsi_target(self, tid, lun, vol_id, **kwargs):
LOG.info(_('Removing iscsi_target for volume: %s') % vol_id)
self._delete_logicalunit(tid, lun, **kwargs)
self._delete_target(tid, **kwargs)
vol_uuid_file = CONF.volume_name_template % vol_id
conf_file = CONF.iet_conf
if os.path.exists(conf_file):
with utils.temporary_chown(conf_file):
try:
iet_conf_text = open(conf_file, 'r+')
full_txt = iet_conf_text.readlines()
new_iet_conf_txt = []
count = 0
for line in full_txt:
if count > 0:
count -= 1
continue
elif re.search(vol_uuid_file, line):
count = 2
continue
else:
new_iet_conf_txt.append(line)
iet_conf_text.seek(0)
iet_conf_text.truncate(0)
iet_conf_text.writelines(new_iet_conf_txt)
finally:
iet_conf_text.close()
def upload_volume(context, image_service, image_meta, volume_path, volume_format="raw"):
image_id = image_meta["id"]
if image_meta["disk_format"] == volume_format:
LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta["disk_format"]))
if os.name == "nt":
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if CONF.image_conversion_dir and not os.path.exists(CONF.image_conversion_dir):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta["disk_format"]))
convert_image(volume_path, tmp, image_meta["disk_format"])
data = qemu_img_info(tmp)
if data.file_format != image_meta["disk_format"]:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s")
% {"f1": image_meta["disk_format"], "f2": data.file_format},
)
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def create_iscsi_target(self, name, tid, lun, path,
chap_auth=None, **kwargs):
# NOTE (jdg): Address bug: 1175207
kwargs.pop('old_name', None)
self._new_target(name, tid, **kwargs)
self._new_logicalunit(tid, lun, path, **kwargs)
if chap_auth is not None:
(type, username, password) = chap_auth.split()
self._new_auth(tid, type, username, password, **kwargs)
conf_file = CONF.iet_conf
if os.path.exists(conf_file):
try:
volume_conf = """
Target %s
%s
Lun 0 Path=%s,Type=%s
""" % (name, chap_auth, path, self._iotype(path))
with utils.temporary_chown(conf_file):
f = open(conf_file, 'a+')
f.write(volume_conf)
f.close()
except exception.ProcessExecutionError as e:
vol_id = name.split(':')[1]
LOG.error(_("Failed to create iscsi target for volume "
"id:%(vol_id)s: %(e)s")
% {'vol_id': vol_id, 'e': str(e)})
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
return tid
def upload_volume(context, image_service, image_meta, volume_path):
image_id = image_meta['id']
if (image_meta['disk_format'] == 'raw'):
LOG.debug("%s was raw, no need to convert to %s" %
(image_id, image_meta['disk_format']))
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was raw, converting to %s" %
(image_id, image_meta['disk_format']))
convert_image(volume_path, tmp, image_meta['disk_format'])
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
os.unlink(tmp)
def _get_available_nvmf_subsystems(self):
__, tmp_file_path = tempfile.mkstemp(prefix='nvmet')
# nvmetcli doesn't support printing to stdout yet,
try:
out, err = nvmcli.save(tmp_file_path)
except putils.ProcessExecutionError:
with excutils.save_and_reraise_exception():
LOG.exception('Error from nvmetcli save')
self._delete_file(tmp_file_path)
# temp file must be readable by this process user
# in order to avoid executing cat as root
with utils.temporary_chown(tmp_file_path):
try:
out = cinder.privsep.path.readfile(tmp_file_path)
except putils.ProcessExecutionError:
with excutils.save_and_reraise_exception():
LOG.exception('Failed to read: %s', tmp_file_path)
self._delete_file(tmp_file_path)
nvmf_subsystems = json.loads(out)
self._delete_file(tmp_file_path)
return nvmf_subsystems
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup['volume_id'])
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw', run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s",
image_id, volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s",
image_id, volume_format, image_meta['disk_format'])
convert_image(volume_path, tmp, image_meta['disk_format'],
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup.volume_id)
snapshot = None
if backup.snapshot_id:
snapshot = objects.Snapshot.get_by_id(context, backup.snapshot_id)
temp_snapshot = None
# NOTE(xyang): If it is to backup from snapshot, back it up
# directly. No need to clean it up.
if snapshot:
volume_path = self.local_path(snapshot)
else:
# NOTE(xyang): If it is not to backup from snapshot, check volume
# status. If the volume status is 'in-use', create a temp snapshot
# from the source volume, backup the temp snapshot, and then clean
# up the temp snapshot; if the volume status is 'available', just
# backup the volume.
previous_status = volume.get('previous_status', None)
if previous_status == "in-use":
temp_snapshot = self._create_temp_snapshot(context, volume)
backup.temp_snapshot_id = temp_snapshot.id
backup.save()
volume_path = self.local_path(temp_snapshot)
else:
volume_path = self.local_path(volume)
try:
with utils.temporary_chown(volume_path):
with open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
if temp_snapshot:
self._delete_temp_snapshot(context, temp_snapshot)
backup.temp_snapshot_id = None
backup.save()
def _open_volume_with_path(path, mode):
try:
with utils.temporary_chown(path):
handle = open(path, mode)
return handle
except Exception:
LOG.error(_LE("Failed to open volume from %(path)s."), {'path': path})
def create_iscsi_target(self, name, tid, lun, path,
chap_auth=None, **kwargs):
self._new_target(name, tid, **kwargs)
self._new_logicalunit(tid, lun, path, **kwargs)
if chap_auth is not None:
(type, username, password) = chap_auth.split()
self._new_auth(tid, type, username, password, **kwargs)
conf_file = FLAGS.iet_conf
if os.path.exists(conf_file):
try:
volume_conf = """
Target %s
%s
Lun 0 Path=%s,Type=fileio
""" % (name, chap_auth, path)
with utils.temporary_chown(conf_file):
f = open(conf_file, 'a+')
f.write(volume_conf)
f.close()
except exception.ProcessExecutionError, e:
vol_id = name.split(':')[1]
LOG.error(_("Failed to create iscsi target for volume "
"id:%(vol_id)s.") % locals())
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
def _run_restore(self, context, backup, volume):
backup_service = self.service.get_backup_driver(context)
properties = utils.brick_get_connector_properties()
secure_enabled = (
self.volume_rpcapi.secure_file_operations_enabled(context,
volume))
attach_info = self._attach_device(context, volume, properties)
try:
device_path = attach_info['device']['path']
if (isinstance(device_path, six.string_types) and
not os.path.isdir(device_path)):
if secure_enabled:
with open(device_path, 'wb') as device_file:
backup_service.restore(backup, volume.id, device_file)
else:
with utils.temporary_chown(device_path):
with open(device_path, 'wb') as device_file:
backup_service.restore(backup, volume.id,
device_file)
# device_path is already file-like so no need to open it
else:
backup_service.restore(backup, volume.id, device_path)
finally:
self._detach_device(context, attach_info, volume, properties,
force=True)
def _run_backup(self, context, backup, volume):
backup_service = self.service.get_backup_driver(context)
properties = utils.brick_get_connector_properties()
backup_dic = self.volume_rpcapi.get_backup_device(context,
backup, volume)
try:
backup_device = backup_dic.get('backup_device')
is_snapshot = backup_dic.get('is_snapshot')
attach_info = self._attach_device(context, backup_device,
properties, is_snapshot)
try:
device_path = attach_info['device']['path']
if isinstance(device_path, six.string_types):
if backup_dic.get('secure_enabled', False):
with open(device_path) as device_file:
backup_service.backup(backup, device_file)
else:
with utils.temporary_chown(device_path):
with open(device_path) as device_file:
backup_service.backup(backup, device_file)
# device_path is already file-like so no need to open it
else:
backup_service.backup(backup, device_path)
finally:
self._detach_device(context, attach_info,
backup_device, properties,
is_snapshot)
finally:
backup = objects.Backup.get_by_id(context, backup.id)
self._cleanup_temp_volumes_snapshots_when_backup_created(
context, backup)
def update_config_file(self, name, tid, path, config_auth):
conf_file = self.iet_conf
vol_id = name.split(':')[1]
# If config file does not exist, create a blank conf file and
# add configuration for the volume on the new file.
if not os.path.exists(conf_file):
try:
utils.execute("truncate", conf_file, "--size=0",
run_as_root=True)
except putils.ProcessExecutionError:
LOG.exception(_LE("Failed to create %(conf)s for volume "
"id:%(vol_id)s"),
{'conf': conf_file, 'vol_id': vol_id})
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
try:
volume_conf = """
Target %s
%s
Lun 0 Path=%s,Type=%s
""" % (name, config_auth, path, self._iotype(path))
with utils.temporary_chown(conf_file):
with open(conf_file, 'a+') as f:
f.write(volume_conf)
except Exception:
LOG.exception(_LE("Failed to update %(conf)s for volume "
"id:%(vol_id)s"),
{'conf': conf_file, 'vol_id': vol_id})
raise exception.ISCSITargetCreateFailed(volume_id=vol_id)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.debug("Begin restore of backup %s." % backup["id"])
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, "wb") as volume_file:
backup_service.restore(backup, volume["id"], volume_file)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.info(_LI('Restoring backup %(backup)s to volume %(volume)s.') %
{'backup': backup['id'], 'volume': volume['name']})
volume_local_path = self.local_path(volume)
with utils.temporary_chown(volume_local_path):
with fileutils.file_open(volume_local_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def test_matching_uid(self, mock_exec, mock_getuid, mock_stat):
mock_stat.return_value.st_uid = 5678
test_filename = 'a_file'
with utils.temporary_chown(test_filename):
pass
mock_getuid.asset_called_once_with()
mock_stat.assert_called_once_with(test_filename)
self.assertFalse(mock_exec.called)
def write_data(device, data):
print("Write data to volume.")
if isinstance(device, six.string_types):
with cinder_utils.temporary_chown(device):
with open(device, 'w') as device_file:
device_file.write(data)
else:
device.write(data)
print("data is written to volume.")
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
LOG.info(
_LI("Restoring backup %(backup)s to volume %(volume)s."), {"backup": backup["id"], "volume": volume.name}
)
volume_local_path = self.local_path(volume)
with utils.temporary_chown(volume_local_path):
with open(volume_local_path, "wb") as volume_file:
backup_service.restore(backup, volume.id, volume_file)
def test_temporary_chown(self):
def fake_execute(*args, **kwargs):
if args[0] == 'chown':
fake_execute.uid = args[1]
self.stubs.Set(utils, 'execute', fake_execute)
with tempfile.NamedTemporaryFile() as f:
with utils.temporary_chown(f.name, owner_uid=2):
self.assertEqual(fake_execute.uid, 2)
self.assertEqual(fake_execute.uid, os.getuid())
def test_restore_backup(self):
volume = {'id': '2', 'name': self.TEST_VOLNAME}
self.mox.StubOutWithMock(utils, 'temporary_chown')
mock_tempchown = mock.MagicMock()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
self.mox.StubOutWithMock(fileutils, 'file_open')
mock_fileopen = mock.MagicMock()
fileutils.file_open(self.TEST_VOLPATH, 'wb').AndReturn(mock_fileopen)
backup = {'id': 123, 'volume_id': volume['id']}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.restore(backup, volume['id'], mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.restore_backup(context, backup, volume,
mock_servicebackup)
def read_data(device):
print("Read 512 bytes from device.")
if isinstance(device, six.string_types):
with cinder_utils.temporary_chown(device):
with open(device, 'r') as device_file:
data = device_file.read(512)
print("data is %s." % data_convert(data))
else:
data = device.read(512)
print("data is %s." % data_convert(data))
def test_supplied_owner_uid(self, mock_exec, mock_getuid, mock_stat):
mock_stat.return_value.st_uid = 5678
test_filename = 'a_file'
with utils.temporary_chown(test_filename, owner_uid=9101):
mock_exec.assert_called_once_with('chown', 9101, test_filename,
run_as_root=True)
self.assertFalse(mock_getuid.called)
mock_stat.assert_called_once_with(test_filename)
calls = [mock.call('chown', 9101, test_filename, run_as_root=True),
mock.call('chown', 5678, test_filename, run_as_root=True)]
mock_exec.assert_has_calls(calls)
def test_get_uid(self, mock_exec, mock_getuid, mock_stat):
mock_stat.return_value.st_uid = 5678
test_filename = 'a_file'
with utils.temporary_chown(test_filename):
mock_exec.assert_called_once_with('chown', 1234, test_filename,
run_as_root=True)
mock_getuid.asset_called_once_with()
mock_stat.assert_called_once_with(test_filename)
calls = [mock.call('chown', 1234, test_filename, run_as_root=True),
mock.call('chown', 5678, test_filename, run_as_root=True)]
mock_exec.assert_has_calls(calls)
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw', run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s",
image_id, volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s",
image_id, volume_format, image_meta['disk_format'])
data = qemu_img_info(volume_path, run_as_root=run_as_root)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file})
out_format = image_meta['disk_format']
# qemu-img accepts 'vpc' as argument for 'vhd 'format and 'parallels'
# as argument for 'ploop'.
if out_format == 'vhd':
out_format = 'vpc'
if out_format == 'ploop':
out_format = 'parallels'
convert_image(volume_path, tmp, out_format,
run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != out_format:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': out_format, 'f2': data.file_format})
with open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
def test_restore_backup(self):
volume = {"id": "2", "name": self.TEST_VOLNAME}
self.mox.StubOutWithMock(utils, "temporary_chown")
mock_tempchown = self.mox.CreateMockAnything()
utils.temporary_chown(self.TEST_VOLPATH).AndReturn(mock_tempchown)
mock_tempchown.__enter__()
mock_tempchown.__exit__(None, None, None)
self.mox.StubOutWithMock(fileutils, "file_open")
mock_fileopen = self.mox.CreateMockAnything()
fileutils.file_open(self.TEST_VOLPATH, "wb").AndReturn(mock_fileopen)
mock_fileopen.__enter__()
mock_fileopen.__exit__(None, None, None)
backup = {"id": 123, "volume_id": volume["id"]}
mock_servicebackup = self.mox.CreateMockAnything()
mock_servicebackup.restore(backup, volume["id"], mox_lib.IgnoreArg())
self.mox.ReplayAll()
self._driver.restore_backup(context, backup, volume, mock_servicebackup)
self.mox.VerifyAll()
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw'):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
data = qemu_img_info(volume_path)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file})
convert_image(volume_path, tmp, image_meta['disk_format'],
bps_limit=CONF.volume_copy_bps_limit)
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def copy_image_to_volume(self, context, volume, image_service, image_id):
"""Fetch the image from image_service and write it to the volume."""
LOG.debug(_('copy_image_to_volume %s.') % volume['name'])
initiator = get_iscsi_initiator()
connector = {}
connector['initiator'] = initiator
iscsi_properties, volume_path = self._attach_volume(
context, volume, connector)
with utils.temporary_chown(volume_path):
with utils.file_open(volume_path, "wb") as image_file:
image_service.download(context, image_id, image_file)
self.terminate_connection(volume, connector)
def restore_backup(self, context, backup, volume, backup_service):
"""Restore an existing backup to a new or existing volume."""
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path, 'wb') as volume_file:
backup_service.restore(backup, volume['id'], volume_file)
def upload_volume(context,
image_service,
image_meta,
volume_path,
volume_format='raw',
run_as_root=True):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s", image_id,
volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with open(volume_path, 'rb') as image_file:
image_service.update(context, image_id,
{"size": image_meta.get("size", 0)},
image_file)
else:
with utils.temporary_chown(volume_path):
with open(volume_path, 'rb') as image_file:
image_service.update(context, image_id,
{"size": image_meta.get("size", 0)},
image_file)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s", image_id, volume_format,
image_meta['disk_format'])
data = qemu_img_info(volume_path, run_as_root=run_as_root)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {
'fmt': fmt,
'backing_file': backing_file
})
out_format = image_meta['disk_format']
# qemu-img accepts 'vpc' as argument for 'vhd 'format and 'parallels'
# as argument for 'ploop'.
if out_format == 'vhd':
out_format = 'vpc'
if out_format == 'ploop':
out_format = 'parallels'
convert_image(volume_path, tmp, out_format, run_as_root=run_as_root)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != out_format:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': out_format,
'f2': data.file_format
})
# qemu_img_info round the size of the of the image created.
# For the upload process we need the exact size of the file size
# so we get it from the opened file
with open(tmp, 'rb') as image_file:
old_file_position = image_file.tell()
image_file.seek(0, os.SEEK_END)
file_size = image_file.tell()
image_file.seek(old_file_position, os.SEEK_SET)
image_service.update(context, image_id, {"size": file_size},
image_file)
def copy_image_to_volume(self, context, volume, image_service, image_id):
"""Fetch the image from image_service and write it to the volume."""
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with utils.file_open(volume_path, "wb") as image_file:
image_service.download(context, image_id, image_file)
def copy_volume_to_image(self, context, volume, image_service, image_id):
"""Copy the volume to the specified image."""
volume_path = self.local_path(volume)
with utils.temporary_chown(volume_path):
with utils.file_open(volume_path) as volume_file:
image_service.update(context, image_id, {}, volume_file)
def upload_volume_to_vgw(context,
image_service,
image_meta,
volume_path,
volume,
vgw_url,
volume_format='raw',
run_as_root=True):
LOG.error('begin time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
image_id = image_meta['id']
volume_id = volume['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as files:
r = requests.post(vgw_url, data=files)
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as files:
r = requests.post(vgw_url, data=files)
#LOG.debug('the request result is %s' %(str(r.status_code)))
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
LOG.error('begin time of convert_image is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
convert_image(volume_path, tmp, image_meta['disk_format'])
LOG.error('end time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp, 'rb') as files:
LOG.error('begin time of upload file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
r = requests.post(vgw_url, data=files)
#LOG.debug('the request result is %' %(str(r.status_code)))
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
LOG.error('end time of upload file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
#todo delete the tmp file
fileutils.delete_if_exists(tmp)
LOG.error('end time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def backup_volume(self, context, backup, backup_service):
"""Create a new backup from an existing volume."""
volume = self.db.volume_get(context, backup.volume_id)
snapshot = None
if backup.snapshot_id:
snapshot = objects.Snapshot.get_by_id(context, backup.snapshot_id)
root_helper = utils.get_root_helper()
temp_zvol = None
temp_zvol_snapshot = None
temp_snapshot = None
# NOTE(xyang): If it is to backup from snapshot, back it up
# directly. No need to clean it up.
if snapshot:
volume_path = self.local_path(snapshot)
else:
# NOTE(xyang): If it is not to backup from snapshot, check volume
# status. If the volume status is 'in-use', create a temp snapshot
# from the source volume, backup the temp snapshot, and then clean
# up the temp snapshot; if the volume status is 'available', just
# backup the volume.
previous_status = volume.get('previous_status', None)
if previous_status == "in-use":
temp_snapshot = self._create_temp_snapshot(context, volume)
backup.temp_snapshot_id = temp_snapshot.id
backup.save()
volume_path = self.local_path(temp_snapshot)
else:
volume_path = self.local_path(volume)
try:
with utils.temporary_chown(volume_path):
with open(volume_path) as volume_file:
backup_service.backup(backup, volume_file)
finally:
# Destroy temporary volume (if exists)
if temp_zvol:
try:
self._execute('zfs',
'destroy',
'-d',
temp_zvol,
root_helper=root_helper,
run_as_root=True)
except processutils.ProcessExecutionError as exc:
exception_message = (_("Failed to delete temporary zvol, "
"error message was: %s") %
six.text_type(exc.stderr))
raise exception.VolumeBackendAPIException(
data=exception_message)
# Destroy temporary snapshot (if exists)
if temp_zvol_snapshot:
try:
self._execute('zfs',
'destroy',
'-d',
temp_zvol_snapshot,
root_helper=root_helper,
run_as_root=True)
except processutils.ProcessExecutionError as exc:
exception_message = (
_("Failed to delete temporary snapshot, "
"error message was: %s") % six.text_type(exc.stderr))
raise exception.VolumeBackendAPIException(
data=exception_message)
def upload_volume(context,
image_service,
image_meta,
volume_path,
volume_format='raw',
run_as_root=True,
compress=True,
store_id=None,
base_image_ref=None):
# NOTE: You probably want to use volume_utils.upload_volume(),
# not this function.
image_id = image_meta['id']
if image_meta.get('container_format') != 'compressed':
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s", image_id,
volume_format, image_meta['disk_format'])
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with open(volume_path, 'rb') as image_file:
image_service.update(context,
image_id, {},
tpool.Proxy(image_file),
store_id=store_id,
base_image_ref=base_image_ref)
else:
with utils.temporary_chown(volume_path):
with open(volume_path, 'rb') as image_file:
image_service.update(context,
image_id, {},
tpool.Proxy(image_file),
store_id=store_id,
base_image_ref=base_image_ref)
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s", image_id, volume_format,
image_meta['disk_format'])
data = qemu_img_info(volume_path, run_as_root=run_as_root)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {
'fmt': fmt,
'backing_file': backing_file
})
out_format = fixup_disk_format(image_meta['disk_format'])
convert_image(volume_path,
tmp,
out_format,
run_as_root=run_as_root,
compress=compress)
data = qemu_img_info(tmp, run_as_root=run_as_root)
if data.file_format != out_format:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': out_format,
'f2': data.file_format
})
# NOTE(ZhengMa): This is used to do image compression on image
# uploading with 'compressed' container_format.
# Compress file 'tmp' in-place
if image_meta.get('container_format') == 'compressed':
LOG.debug("Container_format set to 'compressed', compressing "
"image before uploading.")
accel = accelerator.ImageAccel(tmp, tmp)
accel.compress_img(run_as_root=run_as_root)
with open(tmp, 'rb') as image_file:
image_service.update(context,
image_id, {},
tpool.Proxy(image_file),
store_id=store_id,
base_image_ref=base_image_ref)
def _run_restore(self, context, backup, volume):
orig_key_id = volume.encryption_key_id
backup_service = self.service(context)
properties = utils.brick_get_connector_properties()
secure_enabled = (self.volume_rpcapi.secure_file_operations_enabled(
context, volume))
attach_info = self._attach_device(context, volume, properties)
# NOTE(geguileo): Not all I/O disk operations properly do greenthread
# context switching and may end up blocking the greenthread, so we go
# with native threads proxy-wrapping the device file object.
try:
device_path = attach_info['device']['path']
open_mode = 'rb+' if os.name == 'nt' else 'wb'
if (isinstance(device_path, six.string_types)
and not os.path.isdir(device_path)):
if secure_enabled:
with open(device_path, open_mode) as device_file:
backup_service.restore(backup, volume.id,
tpool.Proxy(device_file))
else:
with utils.temporary_chown(device_path):
with open(device_path, open_mode) as device_file:
backup_service.restore(backup, volume.id,
tpool.Proxy(device_file))
# device_path is already file-like so no need to open it
else:
backup_service.restore(backup, volume.id,
tpool.Proxy(device_path))
except exception.BackupRestoreCancel:
raise
except Exception:
LOG.exception(
'Restoring backup %(backup_id)s to volume '
'%(volume_id)s failed.', {
'backup_id': backup.id,
'volume_id': volume.id
})
raise
finally:
self._detach_device(context,
attach_info,
volume,
properties,
force=True)
# Regardless of whether the restore was successful, do some
# housekeeping to ensure the restored volume's encryption key ID is
# unique, and any previous key ID is deleted. Start by fetching fresh
# info on the restored volume.
restored_volume = objects.Volume.get_by_id(context, volume.id)
restored_key_id = restored_volume.encryption_key_id
if restored_key_id != orig_key_id:
LOG.info(
'Updating encryption key ID for volume %(volume_id)s '
'from backup %(backup_id)s.', {
'volume_id': volume.id,
'backup_id': backup.id
})
key_mgr = key_manager.API(CONF)
if orig_key_id is not None:
LOG.debug('Deleting original volume encryption key ID.')
volume_utils.delete_encryption_key(context, key_mgr,
orig_key_id)
if backup.encryption_key_id is None:
# This backup predates the current code that stores the cloned
# key ID in the backup database. Fortunately, the key ID
# restored from the backup data _is_ a clone of the original
# volume's key ID, so grab it.
LOG.debug('Gleaning backup encryption key ID from metadata.')
backup.encryption_key_id = restored_key_id
backup.save()
# Clone the key ID again to ensure every restored volume has
# a unique key ID. The volume's key ID should not be the same
# as the backup.encryption_key_id (the copy made when the backup
# was first created).
new_key_id = volume_utils.clone_encryption_key(
context, key_mgr, backup.encryption_key_id)
restored_volume.encryption_key_id = new_key_id
restored_volume.save()
else:
LOG.debug(
'Encryption key ID for volume %(volume_id)s already '
'matches encryption key ID in backup %(backup_id)s.', {
'volume_id': volume.id,
'backup_id': backup.id
})
