def cache(self, fetch_func, filename, size=None, *args, **kwargs):
"""Creates image from template.
Ensures that template and image not already exists.
Ensures that base directory exists.
Synchronizes on template fetching.
:fetch_func: Function that creates the base image
Should accept `target` argument.
:filename: Name of the file in the image directory
:size: Size of created image in bytes (optional)
"""
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def fetch_func_sync(target, *args, **kwargs):
# The image may have been fetched while a subsequent
# call was waiting to obtain the lock.
if not os.path.exists(target):
fetch_func(target=target, *args, **kwargs)
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
if not os.path.exists(base_dir):
fileutils.ensure_tree(base_dir)
base = os.path.join(base_dir, filename)
if not self.check_image_exists() or not os.path.exists(base):
self.create_image(fetch_func_sync, base, size,
*args, **kwargs)
if (size and self.preallocate and self._can_fallocate() and
os.access(self.path, os.W_OK)):
utils.execute('fallocate', '-n', '-l', size, self.path)
def write_stored_info(target, field=None, value=None):
"""Write information about an image."""
if not field:
return
info_file = get_info_filename(target)
LOG.info(_LI('Writing stored info to %s'), info_file)
fileutils.ensure_tree(os.path.dirname(info_file))
lock_name = 'info-%s' % os.path.split(target)[-1]
lock_path = os.path.join(CONF.instances_path, 'locks')
@utils.synchronized(lock_name, external=True, lock_path=lock_path)
def write_file(info_file, field, value):
d = {}
if os.path.exists(info_file):
with open(info_file, 'r') as f:
d = _read_possible_json(f.read(), info_file)
d[field] = value
d['%s-timestamp' % field] = time.time()
with open(info_file, 'w') as f:
f.write(jsonutils.dumps(d))
write_file(info_file, field, value)
def write_stored_info(target, field=None, value=None):
"""Write information about an image."""
if not field:
return
info_file = get_info_filename(target)
LOG.info(_LI('Writing stored info to %s'), info_file)
fileutils.ensure_tree(os.path.dirname(info_file))
lock_name = 'info-%s' % os.path.split(target)[-1]
lock_path = os.path.join(CONF.instances_path, 'locks')
@utils.synchronized(lock_name, external=True, lock_path=lock_path)
def write_file(info_file, field, value):
d = {}
if os.path.exists(info_file):
with open(info_file, 'r') as f:
d = _read_possible_json(f.read(), info_file)
d[field] = value
d['%s-timestamp' % field] = time.time()
with open(info_file, 'w') as f:
f.write(jsonutils.dumps(d))
write_file(info_file, field, value)
def cache(self, fetch_func, filename, size=None, *args, **kwargs):
"""Creates image from template.
Ensures that template and image not already exists.
Ensures that base directory exists.
Synchronizes on template fetching.
:fetch_func: Function that creates the base image
Should accept `target` argument.
:filename: Name of the file in the image directory
:size: Size of created image in bytes (optional)
"""
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def fetch_func_sync(target, *args, **kwargs):
# The image may have been fetched while a subsequent
# call was waiting to obtain the lock.
if not os.path.exists(target):
fetch_func(target=target, *args, **kwargs)
base_dir = os.path.join(CONF.instances_path,
CONF.image_cache_subdirectory_name)
if not os.path.exists(base_dir):
fileutils.ensure_tree(base_dir)
base = os.path.join(base_dir, filename)
if not self.check_image_exists() or not os.path.exists(base):
self.create_image(fetch_func_sync, base, size, *args, **kwargs)
if (size and self.preallocate and self._can_fallocate()
and os.access(self.path, os.W_OK)):
utils.execute('fallocate', '-n', '-l', size, self.path)
def _sign_csr(csr_text, ca_folder):
with utils.tempdir() as tmpdir:
inbound = os.path.join(tmpdir, 'inbound.csr')
outbound = os.path.join(tmpdir, 'outbound.csr')
try:
with open(inbound, 'w') as csrfile:
csrfile.write(csr_text)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to write inbound.csr'))
LOG.debug('Flags path: %s', ca_folder)
start = os.getcwd()
# Change working dir to CA
fileutils.ensure_tree(ca_folder)
os.chdir(ca_folder)
utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config',
'./openssl.cnf', '-infiles', inbound)
out, _err = utils.execute('openssl', 'x509', '-in', outbound,
'-serial', '-noout')
serial = string.strip(out.rpartition('=')[2])
os.chdir(start)
with open(outbound, 'r') as crtfile:
return (serial, crtfile.read())
def _sign_csr(csr_text, ca_folder):
with utils.tempdir() as tmpdir:
inbound = os.path.join(tmpdir, 'inbound.csr')
outbound = os.path.join(tmpdir, 'outbound.csr')
try:
with open(inbound, 'w') as csrfile:
csrfile.write(csr_text)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE('Failed to write inbound.csr'))
LOG.debug('Flags path: %s', ca_folder)
start = os.getcwd()
# Change working dir to CA
fileutils.ensure_tree(ca_folder)
os.chdir(ca_folder)
utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config',
'./openssl.cnf', '-infiles', inbound)
out, _err = utils.execute('openssl', 'x509', '-in', outbound,
'-serial', '-noout')
serial = string.strip(out.rpartition('=')[2])
os.chdir(start)
with open(outbound, 'r') as crtfile:
return (serial, crtfile.read())
def resolve_driver_format(self):
"""Return the driver format for self.path.
First checks self.disk_info_path for an entry.
If it's not there, calls self._get_driver_format(), and then
stores the result in self.disk_info_path
See https://bugs.launchpad.net/patron/+bug/1221190
"""
def _dict_from_line(line):
if not line:
return {}
try:
return jsonutils.loads(line)
except (TypeError, ValueError) as e:
msg = (_("Could not load line %(line)s, got error "
"%(error)s") % {
'line': line,
'error': e
})
raise exception.InvalidDiskInfo(reason=msg)
@utils.synchronized(self.disk_info_path,
external=False,
lock_path=self.lock_path)
def write_to_disk_info_file():
# Use os.open to create it without group or world write permission.
fd = os.open(self.disk_info_path, os.O_RDONLY | os.O_CREAT, 0o644)
with os.fdopen(fd, "r") as disk_info_file:
line = disk_info_file.read().rstrip()
dct = _dict_from_line(line)
if self.path in dct:
msg = _("Attempted overwrite of an existing value.")
raise exception.InvalidDiskInfo(reason=msg)
dct.update({self.path: driver_format})
tmp_path = self.disk_info_path + ".tmp"
fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT, 0o644)
with os.fdopen(fd, "w") as tmp_file:
tmp_file.write('%s\n' % jsonutils.dumps(dct))
os.rename(tmp_path, self.disk_info_path)
try:
if (self.disk_info_path is not None
and os.path.exists(self.disk_info_path)):
with open(self.disk_info_path) as disk_info_file:
line = disk_info_file.read().rstrip()
dct = _dict_from_line(line)
for path, driver_format in dct.iteritems():
if path == self.path:
return driver_format
driver_format = self._get_driver_format()
if self.disk_info_path is not None:
fileutils.ensure_tree(os.path.dirname(self.disk_info_path))
write_to_disk_info_file()
except OSError as e:
raise exception.DiskInfoReadWriteFail(reason=six.text_type(e))
return driver_format
def put(self, bucket_name):
path = os.path.abspath(
os.path.join(self.application.directory, bucket_name))
if (not path.startswith(self.application.directory)
or os.path.exists(path)):
self.set_status(403)
return
fileutils.ensure_tree(path)
self.finish()
def put(self, bucket_name):
path = os.path.abspath(os.path.join(
self.application.directory, bucket_name))
if (not path.startswith(self.application.directory) or
os.path.exists(path)):
self.set_status(403)
return
fileutils.ensure_tree(path)
self.finish()
def resolve_driver_format(self):
"""Return the driver format for self.path.
First checks self.disk_info_path for an entry.
If it's not there, calls self._get_driver_format(), and then
stores the result in self.disk_info_path
See https://bugs.launchpad.net/patron/+bug/1221190
"""
def _dict_from_line(line):
if not line:
return {}
try:
return jsonutils.loads(line)
except (TypeError, ValueError) as e:
msg = (_("Could not load line %(line)s, got error "
"%(error)s") %
{'line': line, 'error': e})
raise exception.InvalidDiskInfo(reason=msg)
@utils.synchronized(self.disk_info_path, external=False,
lock_path=self.lock_path)
def write_to_disk_info_file():
# Use os.open to create it without group or world write permission.
fd = os.open(self.disk_info_path, os.O_RDONLY | os.O_CREAT, 0o644)
with os.fdopen(fd, "r") as disk_info_file:
line = disk_info_file.read().rstrip()
dct = _dict_from_line(line)
if self.path in dct:
msg = _("Attempted overwrite of an existing value.")
raise exception.InvalidDiskInfo(reason=msg)
dct.update({self.path: driver_format})
tmp_path = self.disk_info_path + ".tmp"
fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT, 0o644)
with os.fdopen(fd, "w") as tmp_file:
tmp_file.write('%s\n' % jsonutils.dumps(dct))
os.rename(tmp_path, self.disk_info_path)
try:
if (self.disk_info_path is not None and
os.path.exists(self.disk_info_path)):
with open(self.disk_info_path) as disk_info_file:
line = disk_info_file.read().rstrip()
dct = _dict_from_line(line)
for path, driver_format in dct.iteritems():
if path == self.path:
return driver_format
driver_format = self._get_driver_format()
if self.disk_info_path is not None:
fileutils.ensure_tree(os.path.dirname(self.disk_info_path))
write_to_disk_info_file()
except OSError as e:
raise exception.DiskInfoReadWriteFail(reason=six.text_type(e))
return driver_format
def mount_volume(volume, mnt_base, configfile=None):
"""Wraps execute calls for mounting a Quobyte volume"""
fileutils.ensure_tree(mnt_base)
command = ['mount.quobyte', volume, mnt_base]
if configfile:
command.extend(['-c', configfile])
LOG.debug('Mounting volume %s at mount point %s ...', volume, mnt_base)
# Run mount command but do not fail on already mounted exit code
utils.execute(*command, check_exit_code=[0, 4])
LOG.info(_LI('Mounted volume: %s'), volume)
def mount_volume(volume, mnt_base, configfile=None):
"""Wraps execute calls for mounting a Quobyte volume"""
fileutils.ensure_tree(mnt_base)
command = ['mount.quobyte', volume, mnt_base]
if configfile:
command.extend(['-c', configfile])
LOG.debug('Mounting volume %s at mount point %s ...',
volume,
mnt_base)
# Run mount command but do not fail on already mounted exit code
utils.execute(*command, check_exit_code=[0, 4])
LOG.info(_LI('Mounted volume: %s'), volume)
def setup_key_pair(self, context):
key_name = '%s%s' % (context.project_id, CONF.vpn_key_suffix)
try:
keypair_api = compute.api.KeypairAPI()
result, private_key = keypair_api.create_key_pair(
context, context.user_id, key_name)
key_dir = os.path.join(CONF.keys_path, context.user_id)
fileutils.ensure_tree(key_dir)
key_path = os.path.join(key_dir, '%s.pem' % key_name)
with open(key_path, 'w') as f:
f.write(private_key)
except (exception.KeyPairExists, os.error, IOError):
pass
return key_name
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.split(base)[-1]
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def create_ploop_image(base, target, size):
image_path = os.path.join(target, "root.hds")
libvirt_utils.copy_image(base, image_path)
utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
target, image_path)
if size:
dd_path = os.path.join(self.path, "DiskDescriptor.xml")
utils.execute('ploop',
'grow',
'-s',
'%dK' % (size >> 10),
dd_path,
run_as_root=True)
if not os.path.exists(self.path):
if CONF.force_raw_images:
self.pcs_format = "raw"
else:
image_meta = IMAGE_API.get(kwargs["context"],
kwargs["image_id"])
format = image_meta.get("disk_format")
if format == "ploop":
self.pcs_format = "expanded"
elif format == "raw":
self.pcs_format = "raw"
else:
reason = _("PCS doesn't support images in %s format."
" You should either set force_raw_images=True"
" in config or upload an image in ploop"
" or raw format.") % format
raise exception.ImageUnacceptable(
image_id=kwargs["image_id"], reason=reason)
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
if os.path.exists(self.path):
return
fileutils.ensure_tree(self.path)
remove_func = functools.partial(fileutils.delete_if_exists,
remove=shutil.rmtree)
with fileutils.remove_path_on_error(self.path, remove=remove_func):
create_ploop_image(base, self.path, size)
def ensure_ca_filesystem():
"""Ensure the CA filesystem exists."""
ca_dir = ca_folder()
if not os.path.exists(ca_path()):
genrootca_sh_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'CA', 'genrootca.sh'))
start = os.getcwd()
fileutils.ensure_tree(ca_dir)
os.chdir(ca_dir)
try:
utils.execute("sh", genrootca_sh_path)
finally:
os.chdir(start)
def ensure_ca_filesystem():
"""Ensure the CA filesystem exists."""
ca_dir = ca_folder()
if not os.path.exists(ca_path()):
genrootca_sh_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'CA', 'genrootca.sh'))
start = os.getcwd()
fileutils.ensure_tree(ca_dir)
os.chdir(ca_dir)
try:
utils.execute("sh", genrootca_sh_path)
finally:
os.chdir(start)
def setup_key_pair(self, context):
key_name = '%s%s' % (context.project_id, CONF.vpn_key_suffix)
try:
keypair_api = compute.api.KeypairAPI()
result, private_key = keypair_api.create_key_pair(context,
context.user_id,
key_name)
key_dir = os.path.join(CONF.keys_path, context.user_id)
fileutils.ensure_tree(key_dir)
key_path = os.path.join(key_dir, '%s.pem' % key_name)
with open(key_path, 'w') as f:
f.write(private_key)
except (exception.KeyPairExists, os.error, IOError):
pass
return key_name
def create_image(self, prepare_template, base, size, *args, **kwargs):
filename = os.path.split(base)[-1]
@utils.synchronized(filename, external=True, lock_path=self.lock_path)
def create_ploop_image(base, target, size):
image_path = os.path.join(target, "root.hds")
libvirt_utils.copy_image(base, image_path)
utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
target, image_path)
if size:
dd_path = os.path.join(self.path, "DiskDescriptor.xml")
utils.execute('ploop', 'grow', '-s', '%dK' % (size >> 10),
dd_path, run_as_root=True)
if not os.path.exists(self.path):
if CONF.force_raw_images:
self.pcs_format = "raw"
else:
image_meta = IMAGE_API.get(kwargs["context"],
kwargs["image_id"])
format = image_meta.get("disk_format")
if format == "ploop":
self.pcs_format = "expanded"
elif format == "raw":
self.pcs_format = "raw"
else:
reason = _("PCS doesn't support images in %s format."
" You should either set force_raw_images=True"
" in config or upload an image in ploop"
" or raw format.") % format
raise exception.ImageUnacceptable(
image_id=kwargs["image_id"],
reason=reason)
if not os.path.exists(base):
prepare_template(target=base, max_size=size, *args, **kwargs)
self.verify_base_size(base, size)
if os.path.exists(self.path):
return
fileutils.ensure_tree(self.path)
remove_func = functools.partial(fileutils.delete_if_exists,
remove=shutil.rmtree)
with fileutils.remove_path_on_error(self.path, remove=remove_func):
create_ploop_image(base, self.path, size)
def put(self, bucket, object_name):
object_name = urllib.unquote(object_name)
bucket_dir = os.path.abspath(
os.path.join(self.application.directory, bucket))
if (not bucket_dir.startswith(self.application.directory)
or not os.path.isdir(bucket_dir)):
self.set_404()
return
path = self._object_path(bucket, object_name)
if not path.startswith(bucket_dir) or os.path.isdir(path):
self.set_status(403)
return
directory = os.path.dirname(path)
fileutils.ensure_tree(directory)
object_file = open(path, "w")
object_file.write(self.request.body)
object_file.close()
self.set_header('ETag', '"%s"' % utils.get_hash_str(self.request.body))
self.finish()
def put(self, bucket, object_name):
object_name = urllib.unquote(object_name)
bucket_dir = os.path.abspath(os.path.join(
self.application.directory, bucket))
if (not bucket_dir.startswith(self.application.directory) or
not os.path.isdir(bucket_dir)):
self.set_404()
return
path = self._object_path(bucket, object_name)
if not path.startswith(bucket_dir) or os.path.isdir(path):
self.set_status(403)
return
directory = os.path.dirname(path)
fileutils.ensure_tree(directory)
object_file = open(path, "w")
object_file.write(self.request.body)
object_file.close()
self.set_header('ETag',
'"%s"' % utils.get_hash_str(self.request.body))
self.finish()
def __init__(self, root_directory, bucket_depth=0, mapper=None):
versionutils.report_deprecated_feature(
LOG,
_LW('The in tree EC2 API is deprecated as of Kilo release and may '
'be removed in a future release. The stackforge ec2-api '
'project http://git.openstack.org/cgit/stackforge/ec2-api/ '
'is the target replacement for this functionality.')
)
if mapper is None:
mapper = routes.Mapper()
mapper.connect('/',
controller=lambda *a, **kw: RootHandler(self)(*a, **kw))
mapper.connect('/{bucket}/{object_name}',
controller=lambda *a, **kw: ObjectHandler(self)(*a, **kw))
mapper.connect('/{bucket_name}/',
controller=lambda *a, **kw: BucketHandler(self)(*a, **kw))
self.directory = os.path.abspath(root_directory)
fileutils.ensure_tree(self.directory)
self.bucket_depth = bucket_depth
super(S3Application, self).__init__(mapper)
def __init__(self, root_directory, bucket_depth=0, mapper=None):
versionutils.report_deprecated_feature(
LOG,
_LW('The in tree EC2 API is deprecated as of Kilo release and may '
'be removed in a future release. The stackforge ec2-api '
'project http://git.openstack.org/cgit/stackforge/ec2-api/ '
'is the target replacement for this functionality.'))
if mapper is None:
mapper = routes.Mapper()
mapper.connect('/',
controller=lambda *a, **kw: RootHandler(self)(*a, **kw))
mapper.connect('/{bucket}/{object_name}',
controller=lambda *a, **kw: ObjectHandler(self)
(*a, **kw))
mapper.connect('/{bucket_name}/',
controller=lambda *a, **kw: BucketHandler(self)
(*a, **kw))
self.directory = os.path.abspath(root_directory)
fileutils.ensure_tree(self.directory)
self.bucket_depth = bucket_depth
super(S3Application, self).__init__(mapper)
def _add_file(self, basedir, path, data):
filepath = os.path.join(basedir, path)
dirname = os.path.dirname(filepath)
fileutils.ensure_tree(dirname)
with open(filepath, 'wb') as f:
f.write(data)
def _add_file(self, basedir, path, data):
filepath = os.path.join(basedir, path)
dirname = os.path.dirname(filepath)
fileutils.ensure_tree(dirname)
with open(filepath, 'wb') as f:
f.write(data)
def setup(self):
"""Ensure the keychains and folders exist."""
# NOTE(vish): One of the drawbacks of doing this in the api is
# the keys will only be on the api node that launched
# the cloudpipe.
fileutils.ensure_tree(CONF.keys_path)