def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw'):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt':
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
convert_image(volume_path, tmp, image_meta['disk_format'])
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
LOG.error('begin time of COPY_VOLUME_TO_IMAGE is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format=image_meta.get('container_format')
file_name=image_meta.get('id')
full_url=None
if container_format in ['fs_vgw_url','vcloud_vgw_url','aws_vgw_url']:
LOG.debug('get the vgw url')
vgw_url = CONF.vgw.vgw_url.get(container_format)
if vgw_url:
full_url=vgw_url+'/'+file_name
image_utils.upload_volume_to_vgw(context, image_service, image_meta, self.local_path(volume), volume, full_url)
#create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context,
image_service,
image_meta,
tmp)
fileutils.delete_if_exists(tmp)
LOG.error('end time of COPY_VOLUME_TO_IMAGE is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
else:
image_utils.upload_volume(context,
image_service,
image_meta,
self.local_path(volume))
def _do_create_volume(self, volume):
"""Create a volume on given glusterfs_share.
:param volume: volume reference
"""
volume_path = self.local_path(volume)
volume_size = volume['size']
LOG.debug("creating new volume at %s", volume_path)
if os.path.exists(volume_path):
msg = _('file already exists at %s') % volume_path
LOG.error(msg)
raise exception.InvalidVolume(reason=msg)
if self.configuration.nas_volume_prov_type == 'thin':
self._create_qcow2_file(volume_path, volume_size)
else:
try:
self._fallocate(volume_path, volume_size)
except processutils.ProcessExecutionError as exc:
if 'Operation not supported' in exc.stderr:
warnings.warn('Fallocate not supported by current version '
'of glusterfs. So falling back to dd.')
self._create_regular_file(volume_path, volume_size)
else:
fileutils.delete_if_exists(volume_path)
raise
self._set_rw_permissions_for_all(volume_path)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
if not os.path.exists(
self.configuration.provider_image_conversion_dir):
fileutils.ensure_tree(
self.configuration.provider_image_conversion_dir)
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
task_ret = self.adpter.export_volume(
provider_volume_id,
self.configuration.provider_image_conversion_dir,
str(image_meta['id']),
cgw_host_id=self.configuration.cgw_host_id,
cgw_host_ip=self.configuration.cgw_host_ip,
cgw_username=self.configuration.cgw_username,
cgw_certificate=self.configuration.cgw_certificate,
transfer_station=self.configuration.storage_tmp_dir)
if not task_ret:
raise exception_ex.ProviderExportVolumeError
temp_path = os.path.join(
self.configuration.provider_image_conversion_dir,
str(image_meta['id']))
upload_image = temp_path
try:
image_utils.upload_volume(context, image_service, image_meta,
upload_image)
finally:
fileutils.delete_if_exists(upload_image)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
LOG.error('begin time of COPY_VOLUME_TO_IMAGE is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format = image_meta.get('container_format')
file_name = image_meta.get('id')
full_url = None
if container_format in ['fs_vgw_url', 'vcloud_vgw_url', 'aws_vgw_url']:
LOG.debug('get the vgw url')
vgw_url = CONF.vgw.vgw_url.get(container_format)
if vgw_url:
full_url = vgw_url + '/' + file_name
image_utils.upload_volume_to_vgw(context,
image_service, image_meta,
self.local_path(volume), volume,
full_url)
#create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context, image_service, image_meta,
tmp)
fileutils.delete_if_exists(tmp)
LOG.error('end time of COPY_VOLUME_TO_IMAGE is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
else:
image_utils.upload_volume(context, image_service, image_meta,
self.local_path(volume))
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw'):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt':
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
convert_image(volume_path, tmp, image_meta['disk_format'])
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp) as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def _do_create_volume(self, volume):
"""Create a volume on given glusterfs_share.
:param volume: volume reference
"""
volume_path = self.local_path(volume)
volume_size = volume['size']
LOG.debug("creating new volume at %s", volume_path)
if os.path.exists(volume_path):
msg = _('file already exists at %s') % volume_path
LOG.error(msg)
raise exception.InvalidVolume(reason=msg)
if self.configuration.nas_volume_prov_type == 'thin':
self._create_qcow2_file(volume_path, volume_size)
else:
try:
self._fallocate(volume_path, volume_size)
except processutils.ProcessExecutionError as exc:
if 'Operation not supported' in exc.stderr:
warnings.warn('Fallocate not supported by current version '
'of glusterfs. So falling back to dd.')
self._create_regular_file(volume_path, volume_size)
else:
fileutils.delete_if_exists(volume_path)
raise
self._set_rw_permissions_for_all(volume_path)
def delete_volume(self, volume):
"""Deletes a logical volume."""
if not volume['provider_location']:
LOG.warn(
_LW('Volume %s does not have provider_location '
'specified, skipping'), volume['name'])
return
self._ensure_share_mounted(volume['provider_location'])
volume_dir = self._local_volume_dir(volume)
mounted_path = os.path.join(volume_dir,
self.get_active_image_from_info(volume))
self._execute('rm',
'-f',
mounted_path,
run_as_root=self._execute_as_root)
# If an exception (e.g. timeout) occurred during delete_snapshot, the
# base volume may linger around, so just delete it if it exists
base_volume_path = self._local_path_volume(volume)
fileutils.delete_if_exists(base_volume_path)
info_path = self._local_path_volume_info(volume)
fileutils.delete_if_exists(info_path)
def replace_xenserver_image_with_coalesced_vhd(image_file):
with temporary_dir() as tempdir:
extract_targz(image_file, tempdir)
chain = discover_vhd_chain(tempdir)
fix_vhd_chain(chain)
coalesced = coalesce_chain(chain)
fileutils.delete_if_exists(image_file)
rename_file(coalesced, image_file)
def delete_volume(self, volume):
"""Delete a volume."""
# xxx(wangfeng)
#pdb.set_trace()
volume_id = volume['id']
volume_file_name = '%s/%s.vmdk' %(CONF.vcloud.vcloud_volumes_dir,volume_id)
fileutils.delete_if_exists(volume_file_name)
def replace_xenserver_image_with_coalesced_vhd(image_file):
with temporary_dir() as tempdir:
extract_targz(image_file, tempdir)
chain = discover_vhd_chain(tempdir)
fix_vhd_chain(chain)
coalesced = coalesce_chain(chain)
fileutils.delete_if_exists(image_file)
rename_file(coalesced, image_file)
def temporary_file(*args, **kwargs):
tmp = None
try:
tmp = create_temporary_file(*args, **kwargs)
yield tmp
finally:
if tmp:
fileutils.delete_if_exists(tmp)
def temporary_file(*args, **kwargs):
tmp = None
try:
tmp = create_temporary_file(*args, **kwargs)
yield tmp
finally:
if tmp:
fileutils.delete_if_exists(tmp)
def delete_volume(self, volume):
"""Delete a volume."""
# xxx(wangfeng)
#pdb.set_trace()
volume_id = volume['id']
volume_file_name = '%s/%s.vmdk' % (CONF.vcloud.vcloud_volumes_dir,
volume_id)
fileutils.delete_if_exists(volume_file_name)
def test_file_unlinked(self):
mox = self.mox
mox.StubOutWithMock(image_utils, 'create_temporary_file')
mox.StubOutWithMock(fileutils, 'delete_if_exists')
image_utils.create_temporary_file().AndReturn('somefile')
fileutils.delete_if_exists('somefile')
mox.ReplayAll()
with image_utils.temporary_file():
pass
def test_file_unlinked(self):
mox = self.mox
mox.StubOutWithMock(image_utils, 'create_temporary_file')
mox.StubOutWithMock(fileutils, 'delete_if_exists')
image_utils.create_temporary_file().AndReturn('somefile')
fileutils.delete_if_exists('somefile')
mox.ReplayAll()
with image_utils.temporary_file():
pass
def upload_volume(context, image_service, image_meta, volume_path,
volume_format='raw'):
image_id = image_meta['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as image_file:
image_service.update(context, image_id, {}, image_file)
return
if (CONF.image_conversion_dir and not
os.path.exists(CONF.image_conversion_dir)):
os.makedirs(CONF.image_conversion_dir)
fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir)
os.close(fd)
with fileutils.remove_path_on_error(tmp):
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
data = qemu_img_info(volume_path)
backing_file = data.backing_file
fmt = data.file_format
if backing_file is not None:
# Disallow backing files as a security measure.
# This prevents a user from writing an image header into a raw
# volume with a backing file pointing to data they wish to
# access.
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
% {'fmt': fmt, 'backing_file': backing_file})
convert_image(volume_path, tmp, image_meta['disk_format'],
bps_limit=CONF.volume_copy_bps_limit)
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") %
{'f1': image_meta['disk_format'], 'f2': data.file_format})
with fileutils.file_open(tmp, 'rb') as image_file:
image_service.update(context, image_id, {}, image_file)
fileutils.delete_if_exists(tmp)
def test_file_unlinked_on_error(self):
mox = self.mox
mox.StubOutWithMock(image_utils, 'create_temporary_file')
mox.StubOutWithMock(fileutils, 'delete_if_exists')
image_utils.create_temporary_file().AndReturn('somefile')
fileutils.delete_if_exists('somefile')
mox.ReplayAll()
def sut():
with image_utils.temporary_file():
raise test.TestingException()
self.assertRaises(test.TestingException, sut)
def test_file_unlinked_on_error(self):
mox = self.mox
mox.StubOutWithMock(image_utils, 'create_temporary_file')
mox.StubOutWithMock(fileutils, 'delete_if_exists')
image_utils.create_temporary_file().AndReturn('somefile')
fileutils.delete_if_exists('somefile')
mox.ReplayAll()
def sut():
with image_utils.temporary_file():
raise test.TestingException()
self.assertRaises(test.TestingException, sut)
def upload_volume_to_vgw(
context, image_service, image_meta, volume_path, volume, vgw_url, volume_format="raw", run_as_root=True
):
LOG.error("begin time of upload_volume_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
image_id = image_meta["id"]
volume_id = volume["id"]
if image_meta["disk_format"] == volume_format:
LOG.debug("%s was %s, no need to convert to %s" % (image_id, volume_format, image_meta["disk_format"]))
if os.name == "nt" or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, "rb") as files:
r = requests.post(vgw_url, data=files)
if r.status_code != 200:
# LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(reason=("upload the volume %s back_file failed" % volume_id))
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as files:
r = requests.post(vgw_url, data=files)
# LOG.debug('the request result is %s' %(str(r.status_code)))
if r.status_code != 200:
# LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(reason=("upload the volume %s back_file failed" % volume_id))
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s" % (image_id, volume_format, image_meta["disk_format"]))
LOG.error("begin time of convert_image is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
convert_image(volume_path, tmp, image_meta["disk_format"])
LOG.error("end time of upload_volume_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
data = qemu_img_info(tmp)
if data.file_format != image_meta["disk_format"]:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s")
% {"f1": image_meta["disk_format"], "f2": data.file_format},
)
with fileutils.file_open(tmp, "rb") as files:
LOG.error("begin time of upload file is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
r = requests.post(vgw_url, data=files)
# LOG.debug('the request result is %' %(str(r.status_code)))
if r.status_code != 200:
# LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(reason=("upload the volume %s back_file failed" % volume_id))
LOG.error("end time of upload file is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
# todo delete the tmp file
fileutils.delete_if_exists(tmp)
LOG.error("end time of upload_volume_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def _delete_stale_snapshot(self, snapshot):
info_path = self._local_path_volume(snapshot['volume']) + '.info'
snap_info = self._read_info_file(info_path)
if snapshot['id'] in snap_info:
snapshot_file = snap_info[snapshot['id']]
active_file = self.get_active_image_from_info(snapshot['volume'])
snapshot_path = os.path.join(
self._local_volume_dir(snapshot['volume']), snapshot_file)
if (snapshot_file == active_file):
return
LOG.info(_('Deleting stale snapshot: %s') % snapshot['id'])
fileutils.delete_if_exists(snapshot_path)
del(snap_info[snapshot['id']])
self._write_info_file(info_path, snap_info)
def _delete_stale_snapshot(self, snapshot):
info_path = self._local_path_volume(snapshot['volume']) + '.info'
snap_info = self._read_info_file(info_path)
if snapshot['id'] in snap_info:
snapshot_file = snap_info[snapshot['id']]
active_file = self.get_active_image_from_info(snapshot['volume'])
snapshot_path = os.path.join(
self._local_volume_dir(snapshot['volume']), snapshot_file)
if (snapshot_file == active_file):
return
LOG.info(_('Deleting stale snapshot: %s') % snapshot['id'])
fileutils.delete_if_exists(snapshot_path)
del (snap_info[snapshot['id']])
self._write_info_file(info_path, snap_info)
def delete_volume(self, volume):
"""Deletes a logical volume."""
if not volume['provider_location']:
LOG.warn(_('Volume %s does not have provider_location specified, '
'skipping'), volume['name'])
return
self._ensure_share_mounted(volume['provider_location'])
volume_dir = self._local_volume_dir(volume)
mounted_path = os.path.join(volume_dir,
self.get_active_image_from_info(volume))
self._execute('rm', '-f', mounted_path, run_as_root=True)
info_path = self._local_path_volume_info(volume)
fileutils.delete_if_exists(info_path)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
# begin added by liuling
LOG.error('begin time of COPY_VOLUME_TO_IMAGE is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format = image_meta.get('container_format')
file_name = image_meta.get('id')
image_name = image_meta.get('name')
full_url = None
if container_format == 'vgw_url':
LOG.debug('get the vgw url')
#vgw_url = CONF.vgw.vgw_url.get(container_format)
kwargs = {
'auth_url': CONF.keystone_authtoken.keystone_auth_url,
'tenant_name': CONF.keystone_authtoken.tenant_name,
'username': CONF.keystone_authtoken.user_name,
'password': CONF.keystone_authtoken.admin_password,
'insecure': True
}
keystoneclient = kc.Client(**kwargs)
vgw_url = self._get_management_url(keystoneclient,
image_name,
service_type='v2v')
if vgw_url:
full_url = vgw_url + '/' + file_name
image_utils.upload_volume_to_vgw(context,
image_service, image_meta,
self.local_path(volume), volume,
full_url)
#create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context, image_service, image_meta,
tmp)
fileutils.delete_if_exists(tmp)
LOG.error('end time of COPY_VOLUME_TO_IMAGE is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
# end added by liuling
else:
image_utils.upload_volume(context, image_service, image_meta,
self.local_path(volume))
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
disk_format = self.utils.get_supported_format()
temp_vhd_path = os.path.join(self.configuration.image_conversion_dir,
str(image_meta['id']) + '.' + disk_format)
upload_image = temp_vhd_path
try:
self.utils.copy_vhd_disk(self.local_path(volume), temp_vhd_path)
# qemu-img does not yet fully support vhdx format, so we'll first
# convert the image to vhd before attempting upload
if disk_format == 'vhdx':
upload_image = upload_image[:-1]
self.vhdutils.convert_vhd(temp_vhd_path, upload_image)
image_utils.upload_volume(context, image_service, image_meta,
upload_image, 'vpc')
finally:
fileutils.delete_if_exists(temp_vhd_path)
fileutils.delete_if_exists(upload_image)
def _test_copy_volume_to_image(self, supported_format):
drv = self._driver
vol = db_fakes.get_fake_volume_info()
image_meta = db_fakes.get_fake_image_meta()
fake_get_supported_format = lambda x: supported_format
self.stubs.Set(os.path, 'exists', lambda x: False)
self.stubs.Set(drv, 'local_path', self.fake_local_path)
self.stubs.Set(windows_utils.WindowsUtils, 'get_supported_format',
fake_get_supported_format)
self.mox.StubOutWithMock(fileutils, 'ensure_tree')
self.mox.StubOutWithMock(fileutils, 'delete_if_exists')
self.mox.StubOutWithMock(image_utils, 'upload_volume')
self.mox.StubOutWithMock(windows_utils.WindowsUtils, 'copy_vhd_disk')
self.mox.StubOutWithMock(vhdutils.VHDUtils, 'convert_vhd')
fileutils.ensure_tree(CONF.image_conversion_dir)
temp_vhd_path = os.path.join(CONF.image_conversion_dir,
str(image_meta['id']) + "." +
supported_format)
upload_image = temp_vhd_path
windows_utils.WindowsUtils.copy_vhd_disk(self.fake_local_path(vol),
temp_vhd_path)
if supported_format == 'vhdx':
upload_image = upload_image[:-1]
vhdutils.VHDUtils.convert_vhd(temp_vhd_path, upload_image,
constants.VHD_TYPE_DYNAMIC)
image_utils.upload_volume(None, None, image_meta, upload_image, 'vhd')
fileutils.delete_if_exists(temp_vhd_path)
fileutils.delete_if_exists(upload_image)
self.mox.ReplayAll()
drv.copy_volume_to_image(None, vol, None, image_meta)
def delete_volume(self, volume):
"""Deletes a logical volume."""
if not volume["provider_location"]:
LOG.warn(_LW("Volume %s does not have provider_location " "specified, skipping"), volume["name"])
return
self._ensure_share_mounted(volume["provider_location"])
volume_dir = self._local_volume_dir(volume)
mounted_path = os.path.join(volume_dir, self.get_active_image_from_info(volume))
self._execute("rm", "-f", mounted_path, run_as_root=self._execute_as_root)
# If an exception (e.g. timeout) occurred during delete_snapshot, the
# base volume may linger around, so just delete it if it exists
base_volume_path = self._local_path_volume(volume)
fileutils.delete_if_exists(base_volume_path)
info_path = self._local_path_volume_info(volume)
fileutils.delete_if_exists(info_path)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
disk_format = self.utils.get_supported_format()
if not os.path.exists(self.configuration.image_conversion_dir):
fileutils.ensure_tree(self.configuration.image_conversion_dir)
temp_vhd_path = os.path.join(self.configuration.image_conversion_dir, str(image_meta["id"]) + "." + disk_format)
upload_image = temp_vhd_path
try:
self.utils.copy_vhd_disk(self.local_path(volume), temp_vhd_path)
# qemu-img does not yet fully support vhdx format, so we'll first
# convert the image to vhd before attempting upload
if disk_format == "vhdx":
upload_image = upload_image[:-1]
self.vhdutils.convert_vhd(temp_vhd_path, upload_image, constants.VHD_TYPE_DYNAMIC)
image_utils.upload_volume(context, image_service, image_meta, upload_image, "vhd")
finally:
fileutils.delete_if_exists(temp_vhd_path)
fileutils.delete_if_exists(upload_image)
def test_calls(self):
mox = self.mox
mox.StubOutWithMock(image_utils, "temporary_dir")
mox.StubOutWithMock(image_utils, "extract_targz")
mox.StubOutWithMock(image_utils, "discover_vhd_chain")
mox.StubOutWithMock(image_utils, "fix_vhd_chain")
mox.StubOutWithMock(image_utils, "coalesce_chain")
mox.StubOutWithMock(image_utils.os, "unlink")
mox.StubOutWithMock(fileutils, "delete_if_exists")
mox.StubOutWithMock(image_utils, "rename_file")
image_utils.temporary_dir().AndReturn(fake_context("somedir"))
image_utils.extract_targz("image", "somedir")
image_utils.discover_vhd_chain("somedir").AndReturn(["somedir/0.vhd", "somedir/1.vhd"])
image_utils.fix_vhd_chain(["somedir/0.vhd", "somedir/1.vhd"])
image_utils.coalesce_chain(["somedir/0.vhd", "somedir/1.vhd"]).AndReturn("somedir/1.vhd")
fileutils.delete_if_exists("image")
image_utils.rename_file("somedir/1.vhd", "image")
mox.ReplayAll()
image_utils.replace_xenserver_image_with_coalesced_vhd("image")
mox.VerifyAll()
def copy_volume_to_image(self, context, volume, image_service, image_meta):
if not os.path.exists(self.configuration.provider_image_conversion_dir):
fileutils.ensure_tree(self.configuration.provider_image_conversion_dir)
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
task_ret = self.adpter.export_volume(provider_volume_id,
self.configuration.provider_image_conversion_dir,
str(image_meta['id']),
cgw_host_id=self.configuration.cgw_host_id,
cgw_host_ip=self.configuration.cgw_host_ip,
cgw_username=self.configuration.cgw_username,
cgw_certificate=self.configuration.cgw_certificate,
transfer_station=self.configuration.storage_tmp_dir)
if not task_ret:
raise exception_ex.ProviderExportVolumeError
temp_path = os.path.join(self.configuration.provider_image_conversion_dir, str(image_meta['id']))
upload_image = temp_path
try:
image_utils.upload_volume(context, image_service, image_meta,
upload_image)
finally:
fileutils.delete_if_exists(upload_image)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
"""Copy the volume to the specified image."""
LOG.error('begin time of COPY_VOLUME_TO_IMAGE is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format=image_meta.get('container_format')
file_name=image_meta.get('id')
image_name = image_meta.get('name')
full_url=None
if container_format == 'vgw_url':
LOG.debug('get the vgw url')
#vgw_url = CONF.vgw.vgw_url.get(container_format)
kwargs = {
'auth_url': CONF.keystone_authtoken.keystone_auth_url,
'tenant_name':CONF.keystone_authtoken.tenant_name,
'username': CONF.keystone_authtoken.user_name,
'password': CONF.keystone_authtoken.admin_password,
'insecure': True
}
keystoneclient = kc.Client(**kwargs)
vgw_url = self._get_management_url(keystoneclient,image_name, service_type='v2v')
if vgw_url:
full_url=vgw_url+'/'+file_name
image_utils.upload_volume_to_vgw(context, image_service, image_meta, self.local_path(volume), volume, full_url)
#create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context,
image_service,
image_meta,
tmp)
fileutils.delete_if_exists(tmp)
LOG.error('end time of COPY_VOLUME_TO_IMAGE is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
else:
image_utils.upload_volume(context,
image_service,
image_meta,
self.local_path(volume))
def test_calls(self):
mox = self.mox
mox.StubOutWithMock(image_utils, 'temporary_dir')
mox.StubOutWithMock(image_utils, 'extract_targz')
mox.StubOutWithMock(image_utils, 'discover_vhd_chain')
mox.StubOutWithMock(image_utils, 'fix_vhd_chain')
mox.StubOutWithMock(image_utils, 'coalesce_chain')
mox.StubOutWithMock(image_utils.os, 'unlink')
mox.StubOutWithMock(fileutils, 'delete_if_exists')
mox.StubOutWithMock(image_utils, 'rename_file')
image_utils.temporary_dir().AndReturn(fake_context('somedir'))
image_utils.extract_targz('image', 'somedir')
image_utils.discover_vhd_chain('somedir').AndReturn(
['somedir/0.vhd', 'somedir/1.vhd'])
image_utils.fix_vhd_chain(['somedir/0.vhd', 'somedir/1.vhd'])
image_utils.coalesce_chain(
['somedir/0.vhd', 'somedir/1.vhd']).AndReturn('somedir/1.vhd')
fileutils.delete_if_exists('image')
image_utils.rename_file('somedir/1.vhd', 'image')
mox.ReplayAll()
image_utils.replace_xenserver_image_with_coalesced_vhd('image')
mox.VerifyAll()
def test_calls(self):
mox = self.mox
mox.StubOutWithMock(image_utils, 'temporary_dir')
mox.StubOutWithMock(image_utils, 'extract_targz')
mox.StubOutWithMock(image_utils, 'discover_vhd_chain')
mox.StubOutWithMock(image_utils, 'fix_vhd_chain')
mox.StubOutWithMock(image_utils, 'coalesce_chain')
mox.StubOutWithMock(image_utils.os, 'unlink')
mox.StubOutWithMock(fileutils, 'delete_if_exists')
mox.StubOutWithMock(image_utils, 'rename_file')
image_utils.temporary_dir().AndReturn(fake_context('somedir'))
image_utils.extract_targz('image', 'somedir')
image_utils.discover_vhd_chain('somedir').AndReturn(
['somedir/0.vhd', 'somedir/1.vhd'])
image_utils.fix_vhd_chain(['somedir/0.vhd', 'somedir/1.vhd'])
image_utils.coalesce_chain(['somedir/0.vhd', 'somedir/1.vhd'
]).AndReturn('somedir/1.vhd')
fileutils.delete_if_exists('image')
image_utils.rename_file('somedir/1.vhd', 'image')
mox.ReplayAll()
image_utils.replace_xenserver_image_with_coalesced_vhd('image')
mox.VerifyAll()
def copy_volume_to_image(self, context, volume, image_service, image_meta):
LOG.error('begin time of copy_volume_to_image is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format = image_meta.get('container_format')
file_name = image_meta.get('id')
if container_format in ['fs_vgw_url', 'vcloud_vgw_url', 'aws_vgw_url']:
LOG.debug('get the vgw url')
vgw_url = CONF.vgw.vgw_url.get(container_format)
#vgw_url = 'http://162.3.125.52:9999/'
volume_id = volume['id']
#1.get the provider_volume at provider cloud
provider_volume_id = self._get_provider_volumeid_from_volume(
volume)
if not provider_volume_id:
LOG.error('get provider_volume_id of volume %s error' %
volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
provider_volume = self._get_provider_volume(provider_volume_id)
if not provider_volume:
LOG.error(
'get provider_volume of volume %s at provider cloud error'
% volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
origin_provider_volume_state = provider_volume.extra.get(
'attachment_status')
origin_attach_node_id = None
origin_device_name = None
#2.judge if the volume is available
if origin_provider_volume_state is not None:
origin_attach_node_id = provider_volume.extra['instance_id']
origin_device_name = provider_volume.extra['device']
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 50
provider_volume = self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get(
'attachment_status') is None:
break
else:
time.sleep(1)
provider_volume = self._get_provider_volume(
provider_volume_id)
retry_time = retry_time - 1
#3.attach the volume to vgw host
try:
#3.1 get the vgw host
vgw_host = self._get_provider_node(
self.configuration.cgw_host_id)
if not vgw_host:
raise exception_ex.VgwHostNotFound(
Vgw_id=self.configuration.cgw_host_id)
device_name = self._get_next_device_name(vgw_host)
LOG.error('**********************************************')
LOG.error('the volume status %s' % provider_volume.state)
self.adpter.attach_volume(vgw_host, provider_volume,
device_name)
#query volume status
time.sleep(1)
retry_time = 120
provider_volume = self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get(
'attachment_status') == 'attached':
break
else:
time.sleep(1)
provider_volume = self._get_provider_volume(
provider_volume_id)
retry_time = retry_time - 1
except Exception as e:
raise e
time.sleep(5)
conn = rpyc.connect(self.configuration.cgw_host_ip,
int(CONF.vgw.rpc_service_port))
LOG.error('begin time of copy_volume_to_file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
full_file_path = conn.root.copy_volume_to_file(
device_name, file_name, CONF.vgw.store_file_dir)
LOG.error('end time of copy_volume_to_image is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
#todo exception occured clean env
if not full_file_path:
self.adpter.detach_volume(provider_volume)
conn.close()
raise exception_ex.ProviderExportVolumeError(
volume_id=volume_id)
LOG.error('begin time of push_file_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
push_file_result = conn.root.exposed_push_file_to_vgw(
full_file_path, vgw_url)
LOG.error('end time of push_file_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
if not push_file_result:
LOG.error('post file file %s to %s failed' %
(push_file_result, vgw_url))
self.adpter.detach_volume(provider_volume)
conn.close()
raise exception_ex.ProviderExportVolumeError(
volume_id=volume_id)
conn.close()
#create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context, image_service, image_meta,
tmp)
fileutils.delete_if_exists(tmp)
#4.detach form vgw
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 120
provider_volume = self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get(
'attachment_status') is None:
break
else:
time.sleep(1)
provider_volume = self._get_provider_volume(
provider_volume_id)
retry_time = retry_time - 1
LOG.error('**********************************************')
LOG.error('the volume status %s' % provider_volume.state)
#attach the volume back
if origin_provider_volume_state is not None:
origin_attach_node = self._get_provider_node(
origin_attach_node_id)
self.adpter.attach_volume(origin_attach_node, provider_volume,
origin_device_name)
else:
if not os.path.exists(
self.configuration.provider_image_conversion_dir):
fileutils.ensure_tree(
self.configuration.provider_image_conversion_dir)
provider_volume_id = self._get_provider_volumeid_from_volume(
volume)
task_ret = self.adpter.export_volume(
provider_volume_id,
self.configuration.provider_image_conversion_dir,
str(image_meta['id']),
cgw_host_id=self.configuration.cgw_host_id,
cgw_host_ip=self.configuration.cgw_host_ip,
cgw_username=self.configuration.cgw_username,
cgw_certificate=self.configuration.cgw_certificate,
transfer_station=self.configuration.storage_tmp_dir)
if not task_ret:
raise exception_ex.ProviderExportVolumeError
temp_path = os.path.join(
self.configuration.provider_image_conversion_dir,
str(image_meta['id']))
upload_image = temp_path
try:
image_utils.upload_volume(context, image_service, image_meta,
upload_image)
finally:
fileutils.delete_if_exists(upload_image)
LOG.error('end time of copy_volume_to_image is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def fetch_from_localfile_to_raw(
context,
image_service,
image_id,
source_dir,
dest,
blocksize,
user_id=None,
project_id=None,
size=None,
run_as_root=True,
):
LOG.error(
"begin time of fetch_from_localfile_to_raw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
)
image_meta = image_service.show(context, image_id)
file_name = image_meta.get("id")
volume_format = "raw"
# if (CONF.local_file_dir and not
# os.path.exists(CONF.local_file_dir)):
# raise exception.ImageUnacceptable(
# reason=_("the dir of back file of image doesn't exist"),
# image_id=image_id)
tmp = source_dir + "/" + file_name
# tmp='/home/upload/d5414dfe-b6b0-4286-8a23-5867d84f8f27'
# if not os.path.exists(CONF.local_file_dir):
# raise exception.ImageUnacceptable(
# reason=_("the back file of image doesn't exist"),
# image_id=image_id)
if is_xenserver_image(context, image_service, image_id):
replace_xenserver_image_with_coalesced_vhd(tmp)
data = qemu_img_info(tmp)
virt_size = data.virtual_size / units.Gi
# NOTE(xqueralt): If the image virtual size doesn't fit in the
# requested volume there is no point on resizing it because it will
# generate an unusable image.
if size is not None and virt_size > size:
params = {"image_size": virt_size, "volume_size": size}
reason = _("Size is %(image_size)dGB and doesn't fit in a " "volume of size %(volume_size)dGB.") % params
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(reason=_("'qemu-img info' parsing failed."), image_id=image_id)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {"fmt": fmt, "backing_file": backing_file},
)
# NOTE(jdg): I'm using qemu-img convert to write
# to the volume regardless if it *needs* conversion or not
# TODO(avishay): We can speed this up by checking if the image is raw
# and if so, writing directly to the device. However, we need to keep
# check via 'qemu-img info' that what we copied was in fact a raw
# image and not a different format with a backing file, which may be
# malicious.
LOG.debug("%s was %s, converting to %s " % (image_id, fmt, volume_format))
convert_image(tmp, dest, volume_format)
data = qemu_img_info(dest)
if data.file_format != volume_format:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(vol_format)s, but format is " "now %(file_format)s")
% {"vol_format": volume_format, "file_format": data.file_format},
)
try:
LOG.debug("begin delete the image file %s" % tmp)
cmd = ("rm", "-rf", "%s" % tmp)
utils.execute(*cmd, run_as_root=True)
fileutils.delete_if_exists(tmp)
except:
LOG.warning("delete the image %s tmp file failed" % image_id)
LOG.error("end time of fetch_from_localfile_to_raw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def upload_volume_to_vgw(context,
image_service,
image_meta,
volume_path,
volume,
vgw_url,
volume_format='raw',
run_as_root=True):
LOG.error('begin time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
image_id = image_meta['id']
volume_id = volume['id']
if (image_meta['disk_format'] == volume_format):
LOG.debug("%s was %s, no need to convert to %s" %
(image_id, volume_format, image_meta['disk_format']))
if os.name == 'nt' or os.access(volume_path, os.R_OK):
with fileutils.file_open(volume_path, 'rb') as files:
r = requests.post(vgw_url, data=files)
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
else:
with utils.temporary_chown(volume_path):
with fileutils.file_open(volume_path) as files:
r = requests.post(vgw_url, data=files)
#LOG.debug('the request result is %s' %(str(r.status_code)))
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
return
with temporary_file() as tmp:
LOG.debug("%s was %s, converting to %s" %
(image_id, volume_format, image_meta['disk_format']))
LOG.error('begin time of convert_image is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
convert_image(volume_path, tmp, image_meta['disk_format'])
LOG.error('end time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
data = qemu_img_info(tmp)
if data.file_format != image_meta['disk_format']:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(f1)s, but format is now %(f2)s") % {
'f1': image_meta['disk_format'],
'f2': data.file_format
})
with fileutils.file_open(tmp, 'rb') as files:
LOG.error('begin time of upload file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
r = requests.post(vgw_url, data=files)
#LOG.debug('the request result is %' %(str(r.status_code)))
if r.status_code != 200:
#LOG.error('upload file %s to %s failed' %(file_name,vgw_url))
raise exception.ImageUnacceptable(
reason=("upload the volume %s back_file failed" %
volume_id))
LOG.error('end time of upload file is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
#todo delete the tmp file
fileutils.delete_if_exists(tmp)
LOG.error('end time of upload_volume_to_vgw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def fetch_from_localfile_to_raw(context,
image_service,
image_id,
source_dir,
dest,
blocksize,
user_id=None,
project_id=None,
size=None,
run_as_root=True):
LOG.error('begin time of fetch_from_localfile_to_raw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
image_meta = image_service.show(context, image_id)
file_name = image_meta.get('id')
volume_format = 'raw'
# if (CONF.local_file_dir and not
# os.path.exists(CONF.local_file_dir)):
# raise exception.ImageUnacceptable(
# reason=_("the dir of back file of image doesn't exist"),
# image_id=image_id)
tmp = source_dir + '/' + file_name
#tmp='/home/upload/d5414dfe-b6b0-4286-8a23-5867d84f8f27'
# if not os.path.exists(CONF.local_file_dir):
# raise exception.ImageUnacceptable(
# reason=_("the back file of image doesn't exist"),
# image_id=image_id)
if is_xenserver_image(context, image_service, image_id):
replace_xenserver_image_with_coalesced_vhd(tmp)
data = qemu_img_info(tmp)
virt_size = data.virtual_size / units.Gi
# NOTE(xqueralt): If the image virtual size doesn't fit in the
# requested volume there is no point on resizing it because it will
# generate an unusable image.
if size is not None and virt_size > size:
params = {'image_size': virt_size, 'volume_size': size}
reason = _("Size is %(image_size)dGB and doesn't fit in a "
"volume of size %(volume_size)dGB.") % params
raise exception.ImageUnacceptable(image_id=image_id, reason=reason)
fmt = data.file_format
if fmt is None:
raise exception.ImageUnacceptable(
reason=_("'qemu-img info' parsing failed."), image_id=image_id)
backing_file = data.backing_file
if backing_file is not None:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("fmt=%(fmt)s backed by:%(backing_file)s") % {
'fmt': fmt,
'backing_file': backing_file,
})
# NOTE(jdg): I'm using qemu-img convert to write
# to the volume regardless if it *needs* conversion or not
# TODO(avishay): We can speed this up by checking if the image is raw
# and if so, writing directly to the device. However, we need to keep
# check via 'qemu-img info' that what we copied was in fact a raw
# image and not a different format with a backing file, which may be
# malicious.
LOG.debug("%s was %s, converting to %s " % (image_id, fmt, volume_format))
convert_image(tmp, dest, volume_format)
data = qemu_img_info(dest)
if data.file_format != volume_format:
raise exception.ImageUnacceptable(
image_id=image_id,
reason=_("Converted to %(vol_format)s, but format is "
"now %(file_format)s") % {
'vol_format': volume_format,
'file_format': data.file_format
})
try:
LOG.debug('begin delete the image file %s' % tmp)
cmd = ('rm', '-rf', '%s' % tmp)
utils.execute(*cmd, run_as_root=True)
fileutils.delete_if_exists(tmp)
except:
LOG.warning('delete the image %s tmp file failed' % image_id)
LOG.error('end time of fetch_from_localfile_to_raw is %s' %
(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def copy_volume_to_image(self, context, volume, image_service, image_meta):
LOG.error('begin time of copy_volume_to_image is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format=image_meta.get('container_format')
file_name=image_meta.get('id')
if container_format in ['fs_vgw_url','vcloud_vgw_url','aws_vgw_url']:
LOG.debug('get the vgw url')
vgw_url = CONF.vgw.vgw_url.get(container_format)
#vgw_url = 'http://162.3.125.52:9999/'
volume_id = volume['id']
#1.get the provider_volume at provider cloud
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
if not provider_volume_id:
LOG.error('get provider_volume_id of volume %s error' % volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
provider_volume=self._get_provider_volume(provider_volume_id)
if not provider_volume:
LOG.error('get provider_volume of volume %s at provider cloud error' % volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
origin_provider_volume_state= provider_volume.extra.get('attachment_status')
origin_attach_node_id = None
origin_device_name=None
#2.judge if the volume is available
if origin_provider_volume_state is not None:
origin_attach_node_id = provider_volume.extra['instance_id']
origin_device_name = provider_volume.extra['device']
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 50
provider_volume=self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get('attachment_status') is None:
break
else:
time.sleep(1)
provider_volume=self._get_provider_volume(provider_volume_id)
retry_time = retry_time-1
#3.attach the volume to vgw host
try:
#3.1 get the vgw host
vgw_host= self._get_provider_node(self.configuration.cgw_host_id)
if not vgw_host:
raise exception_ex.VgwHostNotFound(Vgw_id=self.configuration.cgw_host_id)
device_name=self._get_next_device_name(vgw_host)
LOG.error('**********************************************')
LOG.error('the volume status %s' %provider_volume.state)
self.adpter.attach_volume(vgw_host, provider_volume,
device_name)
#query volume status
time.sleep(1)
retry_time = 120
provider_volume=self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get('attachment_status') =='attached':
break
else:
time.sleep(1)
provider_volume=self._get_provider_volume(provider_volume_id)
retry_time = retry_time-1
except Exception as e:
raise e
time.sleep(5)
conn=rpyc.connect(self.configuration.cgw_host_ip,int(CONF.vgw.rpc_service_port))
LOG.error('begin time of copy_volume_to_file is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
full_file_path = conn.root.copy_volume_to_file(device_name,file_name,CONF.vgw.store_file_dir)
LOG.error('end time of copy_volume_to_image is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
#todo exception occured clean env
if not full_file_path:
self.adpter.detach_volume(provider_volume)
conn.close()
raise exception_ex.ProviderExportVolumeError(volume_id=volume_id)
LOG.error('begin time of push_file_to_vgw is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
push_file_result =conn.root.exposed_push_file_to_vgw(full_file_path,vgw_url)
LOG.error('end time of push_file_to_vgw is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
if not push_file_result:
LOG.error('post file file %s to %s failed' %(push_file_result,vgw_url))
self.adpter.detach_volume(provider_volume)
conn.close()
raise exception_ex.ProviderExportVolumeError(volume_id=volume_id)
conn.close()
#create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context,
image_service,
image_meta,
tmp)
fileutils.delete_if_exists(tmp)
#4.detach form vgw
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 120
provider_volume=self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get('attachment_status') is None:
break
else:
time.sleep(1)
provider_volume=self._get_provider_volume(provider_volume_id)
retry_time = retry_time-1
LOG.error('**********************************************')
LOG.error('the volume status %s' %provider_volume.state)
#attach the volume back
if origin_provider_volume_state is not None:
origin_attach_node = self._get_provider_node(origin_attach_node_id)
self.adpter.attach_volume(origin_attach_node, provider_volume,
origin_device_name)
else:
if not os.path.exists(self.configuration.provider_image_conversion_dir):
fileutils.ensure_tree(self.configuration.provider_image_conversion_dir)
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
task_ret = self.adpter.export_volume(provider_volume_id,
self.configuration.provider_image_conversion_dir,
str(image_meta['id']),
cgw_host_id=self.configuration.cgw_host_id,
cgw_host_ip=self.configuration.cgw_host_ip,
cgw_username=self.configuration.cgw_username,
cgw_certificate=self.configuration.cgw_certificate,
transfer_station=self.configuration.storage_tmp_dir)
if not task_ret:
raise exception_ex.ProviderExportVolumeError
temp_path = os.path.join(self.configuration.provider_image_conversion_dir, str(image_meta['id']))
upload_image = temp_path
try:
image_utils.upload_volume(context, image_service, image_meta,
upload_image)
finally:
fileutils.delete_if_exists(upload_image)
LOG.error('end time of copy_volume_to_image is %s' %(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def temporary_file():
try:
tmp = create_temporary_file()
yield tmp
finally:
fileutils.delete_if_exists(tmp)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
container_format = image_meta.get('container_format')
#if container_format == 'vgw_url':
if container_format == 'bare':
try:
# 1.get the hws volume id
cascaded_volume_id = volume['id']
hws_volume_id = self.db_manager.get_cascaded_volume_id(cascaded_volume_id)
if not hws_volume_id:
msg = 'get hws volume id error, cascaded id: %s' % cascaded_volume_id
LOG.error(msg)
raise Exception('get hws volume id error')
# 2. get the hws_volume status
volume_detail_rsp = self._get_volume_detail(hws_volume_id)
status = volume_detail_rsp['body']['volume']['status']
# attachments = volume_detail_rsp['body']['volume']['attachments']
# attach_num = len(attachments)
# origin_instance_id = None
# attach_back = False
# 3. detach volume from origin instance
# if status == 'in-use':
# if attach_num != 1:
# msg = 'hws_v2v: get attachments info error, num: %s' % attach_num
# LOG.error(msg)
# raise Exception(msg)
# origin_instance_id = attachments[0]['server_id']
# # volume can only be detached when sever stop
# self._stop_server(origin_instance_id)
# self._detach_volume(origin_instance_id, hws_volume_id)
# attach_back = True
# volume_detail_rsp = self._get_volume_detail(hws_volume_id)
# status = volume_detail_rsp['body']['status']
# 4. attach volume to hws v2v gateway host
if status != 'available':
msg = 'attach volume to local v2v gateway host error, status : %s, cascaded_volume_id: %s, ' \
'hws_volume_id %s' % (status, cascaded_volume_id, hws_volume_id)
LOG.error(msg)
raise Exception('attach volume to local v2v gateway failed')
hws_vgw_instance_id = CONF.hws_vgw.hws_instance_id
# if not hws_vgw_instance_id:
# LOG.error(
# 'hws_v2v: get cascaded v2v gateway instance id error: %s' % CONF.hws_vgw.cascaded_instance_id)
# raise Exception('hws_v2v: get cascaded v2v gateway instance error.')
dev_name = self._get_instance_next_devname(hws_vgw_instance_id)
self._attach_volume(hws_vgw_instance_id, hws_volume_id, dev_name)
# 5. copy volume to file
self._copy_volume_to_file(image_meta, dev_name)
# 6. copy file to remote v2v gateway
# self._copy_file_to_remote_vgw(image_meta)
# 7. create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context,
image_service,
image_meta,
tmp)
fileutils.delete_if_exists(tmp)
# 8. detach volume from hws v2v gateway
self._stop_server(hws_vgw_instance_id)
self._detach_volume(hws_vgw_instance_id, hws_volume_id)
self._power_on(hws_vgw_instance_id)
finally:
attach_back = True
def copy_volume_to_image(self, context, volume, image_service, image_meta):
container_format = image_meta.get('container_format')
#if container_format == 'vgw_url':
if container_format == 'bare':
try:
# 1.get the hws volume id
cascaded_volume_id = volume['id']
hws_volume_id = self.db_manager.get_cascaded_volume_id(
cascaded_volume_id)
if not hws_volume_id:
msg = 'get hws volume id error, cascaded id: %s' % cascaded_volume_id
LOG.error(msg)
raise Exception('get hws volume id error')
# 2. get the hws_volume status
volume_detail_rsp = self._get_volume_detail(hws_volume_id)
status = volume_detail_rsp['body']['volume']['status']
# attachments = volume_detail_rsp['body']['volume']['attachments']
# attach_num = len(attachments)
# origin_instance_id = None
# attach_back = False
# 3. detach volume from origin instance
# if status == 'in-use':
# if attach_num != 1:
# msg = 'hws_v2v: get attachments info error, num: %s' % attach_num
# LOG.error(msg)
# raise Exception(msg)
# origin_instance_id = attachments[0]['server_id']
# # volume can only be detached when sever stop
# self._stop_server(origin_instance_id)
# self._detach_volume(origin_instance_id, hws_volume_id)
# attach_back = True
# volume_detail_rsp = self._get_volume_detail(hws_volume_id)
# status = volume_detail_rsp['body']['status']
# 4. attach volume to hws v2v gateway host
if status != 'available':
msg = 'attach volume to local v2v gateway host error, status : %s, cascaded_volume_id: %s, ' \
'hws_volume_id %s' % (status, cascaded_volume_id, hws_volume_id)
LOG.error(msg)
raise Exception(
'attach volume to local v2v gateway failed')
hws_vgw_instance_id = CONF.hws_vgw.hws_instance_id
# if not hws_vgw_instance_id:
# LOG.error(
# 'hws_v2v: get cascaded v2v gateway instance id error: %s' % CONF.hws_vgw.cascaded_instance_id)
# raise Exception('hws_v2v: get cascaded v2v gateway instance error.')
dev_name = self._get_instance_next_devname(hws_vgw_instance_id)
self._attach_volume(hws_vgw_instance_id, hws_volume_id,
dev_name)
# 5. copy volume to file
self._copy_volume_to_file(image_meta, dev_name)
# 6. copy file to remote v2v gateway
# self._copy_file_to_remote_vgw(image_meta)
# 7. create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context, image_service,
image_meta, tmp)
fileutils.delete_if_exists(tmp)
# 8. detach volume from hws v2v gateway
self._stop_server(hws_vgw_instance_id)
self._detach_volume(hws_vgw_instance_id, hws_volume_id)
self._power_on(hws_vgw_instance_id)
finally:
attach_back = True
def _delete(self, path):
fileutils.delete_if_exists(path)
def _delete(self, path):
fileutils.delete_if_exists(path)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
container_format=image_meta.get('container_format')
#if container_format in ['az01_vgw_url','az11_vgw_url','az31_vgw_url']:
if True:
#vgw_url = self.configuration.container_format
vgw_url = 'http://162.3.114.62:9999/'
volume_id = volume['id']
#1.get the provider_volume at provider cloud
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
if not provider_volume_id:
LOG.error('get provider_volume_id of volume %s error' % volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
provider_volume=self._get_provider_volume(provider_volume_id)
if not provider_volume:
LOG.error('get provider_volume of volume %s at provider cloud error' % volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
origin_provider_volume_state= provider_volume.state
origin_attach_node_id = None
origin_device_name=None
#2.judge if the volume is available
if provider_volume.state != StorageVolumeState.AVAILABLE:
origin_attach_node_id = provider_volume.extra['instance_id']
origin_device_name = provider_volume.extra['device']
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 10
provider_volume=self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.state == StorageVolumeState.AVAILABLE:
break
else:
time.sleep(1)
provider_volume=self._get_provider_volume(provider_volume_id)
retry_time = retry_time-1
#3.attach the volume to vgw host
try:
#3.1 get the vgw host
vgw_host= self._get_provider_node(self.configuration.cgw_host_id)
if not vgw_host:
raise exception_ex.VgwHostNotFound(Vgw_id=self.configuration.cgw_host_id)
device_name=self._get_next_device_name(vgw_host)
self.compute_adapter.attach_volume(vgw_host, provider_volume,
device_name)
#query volume status
time.sleep(1)
retry_time = 10
provider_volume=self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.state == StorageVolumeState.INUSE:
break
else:
time.sleep(1)
provider_volume=self._get_provider_volume(provider_volume_id)
retry_time = retry_time-1
except Exception as e:
raise e
conn=rpyc.connect(self.configuration.cgw_host_ip,'1111')
full_file_path = conn.root.copy_volume_to_file(device_name,volume_id)
if not full_file_path:
raise exception_ex.ProviderExportVolumeError(volume_id=volume_id)
push_file_result =conn.root.exposed_push_file_to_vgw(full_file_path,vgw_url)
if not push_file_result:
LOG.error('post file file %s to %s failed' %(push_file_result,vgw_url))
raise exception_ex.ProviderExportVolumeError(volume_id=volume_id)
conn.close()
image_utils.upload_volume(context,
image_service,
image_meta,
"/home/upload/volume-empty")
#4.detach form vgw
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 10
provider_volume=self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.state == StorageVolumeState.AVAILABLE:
break
else:
time.sleep(1)
provider_volume=self._get_provider_volume(provider_volume_id)
retry_time = retry_time-1
#attach the volume back
# if origin_provider_volume_state != StorageVolumeState.AVAILABLE:
# origin_attach_node = self._get_provider_node(origin_attach_node_id)
#
# self.adapter.attach_volume(origin_attach_node, provider_volume,
# origin_device_name)
else:
if not os.path.exists(self.configuration.provider_image_conversion_dir):
fileutils.ensure_tree(self.configuration.provider_image_conversion_dir)
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
task_ret = self.adpter.export_volume(provider_volume_id,
self.configuration.provider_image_conversion_dir,
str(image_meta['id']),
cgw_host_id=self.configuration.cgw_host_id,
cgw_host_ip=self.configuration.cgw_host_ip,
cgw_username=self.configuration.cgw_username,
cgw_certificate=self.configuration.cgw_certificate,
transfer_station=self.configuration.storage_tmp_dir)
if not task_ret:
raise exception_ex.ProviderExportVolumeError
temp_path = os.path.join(self.configuration.provider_image_conversion_dir, str(image_meta['id']))
upload_image = temp_path
try:
image_utils.upload_volume(context, image_service, image_meta,
upload_image)
finally:
fileutils.delete_if_exists(upload_image)
def temporary_file():
try:
tmp = create_temporary_file()
yield tmp
finally:
fileutils.delete_if_exists(tmp)
def copy_volume_to_image(self, context, volume, image_service, image_meta):
LOG.error("begin time of copy_volume_to_image is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
container_format = image_meta.get("container_format")
image_name = image_meta.get("name")
file_name = image_meta.get("id")
if container_format == "vgw_url":
LOG.debug("get the vgw url")
# vgw_url = CONF.vgw.vgw_url.get(container_format)
kwargs = {
"auth_url": CONF.keystone_authtoken.keystone_auth_url,
"tenant_name": CONF.keystone_authtoken.tenant_name,
"username": CONF.keystone_authtoken.user_name,
"password": CONF.keystone_authtoken.admin_password,
"insecure": True,
}
keystoneclient = kc.Client(**kwargs)
vgw_url = self._get_management_url(keystoneclient, image_name, service_type="v2v")
# vgw_url = 'http://162.3.125.52:9999/'
volume_id = volume["id"]
# 1.get the provider_volume at provider cloud
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
if not provider_volume_id:
LOG.error("get provider_volume_id of volume %s error" % volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
provider_volume = self._get_provider_volume(provider_volume_id)
if not provider_volume:
LOG.error("get provider_volume of volume %s at provider cloud error" % volume_id)
raise exception_ex.ProviderVolumeNotFound(volume_id=volume_id)
origin_provider_volume_state = provider_volume.extra.get("attachment_status")
LOG.error("the origin_provider_volume_info is %s" % str(provider_volume.__dict__))
origin_attach_node_id = None
origin_device_name = None
# 2.judge if the volume is available
if origin_provider_volume_state is not None:
origin_attach_node_id = provider_volume.extra["instance_id"]
origin_device_name = provider_volume.extra["device"]
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 90
provider_volume = self._get_provider_volume(provider_volume_id)
LOG.error("the after detach _volume_info is %s" % str(provider_volume.__dict__))
while retry_time > 0:
if provider_volume and provider_volume.extra.get("attachment_status") is None:
break
else:
time.sleep(2)
provider_volume = self._get_provider_volume(provider_volume_id)
LOG.error(
"the after detach _volume_info is %s,the retry_time is %s"
% (str(provider_volume.__dict__), str(retry_time))
)
retry_time = retry_time - 1
# 3.attach the volume to vgw host
try:
# 3.1 get the vgw host
vgw_host = self._get_provider_node(self.configuration.cgw_host_id)
if not vgw_host:
raise exception_ex.VgwHostNotFound(Vgw_id=self.configuration.cgw_host_id)
device_name = self._get_next_device_name(vgw_host)
LOG.error("**********************************************")
LOG.error("the volume status %s" % provider_volume.state)
self.adpter.attach_volume(vgw_host, provider_volume, device_name)
# query volume status
time.sleep(1)
retry_time = 120
provider_volume = self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get("attachment_status") == "attached":
break
else:
time.sleep(2)
provider_volume = self._get_provider_volume(provider_volume_id)
retry_time = retry_time - 1
except Exception as e:
raise e
time.sleep(5)
conn = rpyc.connect(self.configuration.cgw_host_ip, int(CONF.vgw.rpc_service_port))
LOG.error(
"begin time of copy_volume_to_file is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
)
full_file_path = conn.root.copy_volume_to_file(device_name, file_name, CONF.vgw.store_file_dir)
LOG.error("end time of copy_volume_to_image is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
# todo exception occured clean env
if not full_file_path:
self.adpter.detach_volume(provider_volume)
conn.close()
raise exception_ex.ProviderExportVolumeError(volume_id=volume_id)
LOG.error("begin time of push_file_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
push_file_result = conn.root.exposed_push_file_to_vgw(full_file_path, vgw_url)
LOG.error("end time of push_file_to_vgw is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
if not push_file_result:
LOG.error("post file file %s to %s failed" % (push_file_result, vgw_url))
self.adpter.detach_volume(provider_volume)
conn.close()
raise exception_ex.ProviderExportVolumeError(volume_id=volume_id)
conn.close()
# create a empty file to glance
with image_utils.temporary_file() as tmp:
image_utils.upload_volume(context, image_service, image_meta, tmp)
fileutils.delete_if_exists(tmp)
# 4.detach form vgw
self.adpter.detach_volume(provider_volume)
time.sleep(1)
retry_time = 120
provider_volume = self._get_provider_volume(provider_volume_id)
while retry_time > 0:
if provider_volume and provider_volume.extra.get("attachment_status") is None:
break
else:
time.sleep(2)
provider_volume = self._get_provider_volume(provider_volume_id)
retry_time = retry_time - 1
LOG.error("**********************************************")
LOG.error("the volume status %s" % provider_volume.state)
# attach the volume back
if origin_provider_volume_state is not None:
origin_attach_node = self._get_provider_node(origin_attach_node_id)
self.adpter.attach_volume(origin_attach_node, provider_volume, origin_device_name)
else:
if not os.path.exists(self.configuration.provider_image_conversion_dir):
fileutils.ensure_tree(self.configuration.provider_image_conversion_dir)
provider_volume_id = self._get_provider_volumeid_from_volume(volume)
task_ret = self.adpter.export_volume(
provider_volume_id,
self.configuration.provider_image_conversion_dir,
str(image_meta["id"]),
cgw_host_id=self.configuration.cgw_host_id,
cgw_host_ip=self.configuration.cgw_host_ip,
cgw_username=self.configuration.cgw_username,
cgw_certificate=self.configuration.cgw_certificate,
transfer_station=self.configuration.storage_tmp_dir,
)
if not task_ret:
raise exception_ex.ProviderExportVolumeError
temp_path = os.path.join(self.configuration.provider_image_conversion_dir, str(image_meta["id"]))
upload_image = temp_path
try:
image_utils.upload_volume(context, image_service, image_meta, upload_image)
finally:
fileutils.delete_if_exists(upload_image)
LOG.error("end time of copy_volume_to_image is %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
