def make_drive(self, path):
"""Make the config drive.
:param path: the path to place the config drive image at
:raises ProcessExecuteError if a helper process has failed.
"""
if self.fs_type is None: # a raw image
LOG.debug("Unpacking compressed config drive image")
buffer = StringIO.StringIO(self.compressed_fs)
with gzip.GzipFile(fileobj=buffer, mode='rb') \
as gbuffer:
# TODO(ijw): danger of memory consumption
decompressed = gbuffer.read()
# write the contents as a local disk file
with open(path, 'wb') as conf_drive:
conf_drive.write(decompressed)
# device_type previously set
elif self.fs_type == 'iso9660':
with utils.tempdir() as tmpdir:
self._write_md_files(tmpdir)
self._make_iso9660(path, tmpdir)
self.device_type = 'cdrom'
elif self.fs_type == 'vfat':
with utils.tempdir() as tmpdir:
self._write_md_files(tmpdir)
self._make_vfat(path, tmpdir)
self.device_type = 'disk'
else:
raise exception.ConfigDriveUnknownFormatEx(
format=self.fs_type)
LOG.debug("Config drive device type: %s" % (self.device_type))
def ssh_encrypt_text(ssh_public_key, text):
"""Encrypt text with an ssh public key.
Requires recent ssh-keygen binary in addition to openssl binary.
"""
with utils.tempdir() as tmpdir:
sshkey = os.path.abspath(os.path.join(tmpdir, 'ssh.key'))
with open(sshkey, 'w') as f:
f.write(ssh_public_key)
sslkey = os.path.abspath(os.path.join(tmpdir, 'ssl.key'))
try:
# NOTE(vish): -P is to skip prompt on bad keys
out, _err = utils.execute('ssh-keygen',
'-P', '',
'-e',
'-f', sshkey,
'-m', 'PKCS8')
with open(sslkey, 'w') as f:
f.write(out)
enc, _err = utils.execute('openssl',
'rsautl',
'-encrypt',
'-pubin',
'-inkey', sslkey,
'-keyform', 'PEM',
process_input=text)
return enc
except exception.ProcessExecutionError as exc:
raise exception.EncryptionFailure(reason=exc.stderr)
def test_compute_manager(self):
was = {"called": False}
def fake_get_all_by_filters(context, *args, **kwargs):
was["called"] = True
return [
{
"image_ref": "1",
"host": CONF.host,
"name": "instance-1",
"uuid": "123",
"vm_state": "",
"task_state": "",
},
{
"image_ref": "1",
"host": CONF.host,
"name": "instance-2",
"uuid": "456",
"vm_state": "",
"task_state": "",
},
]
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.stubs.Set(db, "instance_get_all_by_filters", fake_get_all_by_filters)
compute = importutils.import_object(CONF.compute_manager)
self.flags(use_local=True, group="conductor")
compute.conductor_api = conductor.API()
compute._run_image_cache_manager_pass(None)
self.assertTrue(was["called"])
def test_run_image_cache_manager_pass(self, mock_instance_list):
def fake_instances(ctxt):
instances = []
for x in range(2):
instances.append(
fake_instance.fake_db_instance(
image_ref=uuids.fake_image_ref,
uuid=getattr(uuids, "instance_%s" % x),
name="instance-%s" % x,
vm_state="",
task_state="",
)
)
return objects.instance._make_instance_list(ctxt, objects.InstanceList(), instances, None)
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
ctxt = context.get_admin_context()
mock_instance_list.return_value = fake_instances(ctxt)
compute = importutils.import_object(CONF.compute_manager)
self.flags(use_local=True, group="conductor")
compute.conductor_api = conductor.API()
compute._run_image_cache_manager_pass(ctxt)
filters = {"host": ["fake-mini"], "deleted": False, "soft_deleted": True}
mock_instance_list.assert_called_once_with(ctxt, filters, expected_attrs=[], use_slave=True)
def _make_vfat(self, path, tmpdir):
# NOTE(mikal): This is a little horrible, but I couldn't find an
# equivalent to genisoimage for vfat filesystems.
with open(path, 'wb') as f:
f.truncate(CONFIGDRIVESIZE_BYTES)
utils.mkfs('vfat', path, label='config-2')
with utils.tempdir() as mountdir:
mounted = False
try:
_, err = utils.trycmd(
'mount', '-o', 'loop,uid=%d,gid=%d' % (os.getuid(),
os.getgid()),
path,
mountdir,
run_as_root=True)
if err:
raise exception.ConfigDriveMountFailed(operation='mount',
error=err)
mounted = True
# NOTE(mikal): I can't just use shutils.copytree here,
# because the destination directory already
# exists. This is annoying.
for ent in os.listdir(tmpdir):
shutil.copytree(os.path.join(tmpdir, ent),
os.path.join(mountdir, ent))
finally:
if mounted:
utils.execute('umount', mountdir, run_as_root=True)
def test_verify_checksum_invalid_json(self):
img = {'container_format': 'ami', 'id': '42'}
self.flags(checksum_base_images=True)
with self._intercept_log_messages() as stream:
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname, info_fname, testdata = self._make_checksum(tmpdir)
# Checksum is invalid, and not json
f = open(info_fname, 'w')
f.write('banana')
f.close()
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(
img, fname, create_if_missing=False)
self.assertFalse(res)
log = stream.getvalue()
# NOTE(mikal): this is a skip not a fail because the file is
# present, but is not in valid json format and therefore is
# skipped.
self.assertNotEqual(log.find('image verification skipped'), -1)
def test_modified_policy_reloads(self):
with utils.tempdir() as tmpdir:
tmpfilename = os.path.join(tmpdir, 'inject_files')
self.flags(inject_content_file=tmpfilename)
inject_files.reset()
with open(tmpfilename, "w") as policyfile:
policyfile.write("""{"inject_files":
[
{
"path":"test_path",
"contents":"test_content"
}
]
}""")
inject_files_dict = inject_files.get_inject_files()
expect_dict = {
"inject_files": [
{
"path": "test_path",
"contents": "test_content"
}
]
}
self.assertEqual(expect_dict, inject_files_dict)
def test_verify_checksum_invalid_repaired(self):
with utils.tempdir() as tmpdir:
image_cache_manager, fname = (
self._check_body(tmpdir, "csum invalid, not json"))
res = image_cache_manager._verify_checksum(
self.img, fname, create_if_missing=True)
self.assertIsNone(res)
def test_remove_base_file_dne(self):
# This test is solely to execute the "does not exist" code path. We
# don't expect the method being tested to do anything in this case.
with utils.tempdir() as tmpdir:
fname = os.path.join(tmpdir, 'aaa')
image_cache_manager = imagecache.ImageCacheManager()
image_cache_manager._remove_base_file(fname)
def _sign_csr(csr_text, ca_folder):
with utils.tempdir() as tmpdir:
inbound = os.path.join(tmpdir, "inbound.csr")
outbound = os.path.join(tmpdir, "outbound.csr")
try:
with open(inbound, "w") as csrfile:
csrfile.write(csr_text)
except IOError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to write inbound.csr"))
LOG.debug("Flags path: %s", ca_folder)
start = os.getcwd()
# Change working dir to CA
fileutils.ensure_tree(ca_folder)
os.chdir(ca_folder)
utils.execute("openssl", "ca", "-batch", "-out", outbound, "-config", "./openssl.cnf", "-infiles", inbound)
out, _err = utils.execute("openssl", "x509", "-in", outbound, "-serial", "-noout")
serial = string.strip(out.rpartition("=")[2])
os.chdir(start)
with open(outbound, "r") as crtfile:
return (serial, crtfile.read())
def get_encoded_zip(self, project_id):
# Make a payload.zip
with utils.tempdir() as tmpdir:
filename = "payload.zip"
zippath = os.path.join(tmpdir, filename)
z = zipfile.ZipFile(zippath, "w", zipfile.ZIP_DEFLATED)
boot_script = _load_boot_script()
# genvpn, sign csr
crypto.generate_vpn_files(project_id)
z.writestr('autorun.sh', boot_script)
crl = os.path.join(crypto.ca_folder(project_id), 'crl.pem')
z.write(crl, 'crl.pem')
server_key = os.path.join(crypto.ca_folder(project_id),
'server.key')
z.write(server_key, 'server.key')
ca_crt = os.path.join(crypto.ca_path(project_id))
z.write(ca_crt, 'ca.crt')
server_crt = os.path.join(crypto.ca_folder(project_id),
'server.crt')
z.write(server_crt, 'server.crt')
z.close()
zippy = open(zippath, "r")
# NOTE(vish): run instances expects encoded userdata, it is decoded
# in the get_metadata_call. autorun.sh also decodes the zip file,
# hence the double encoding.
encoded = zippy.read().encode("base64").encode("base64")
zippy.close()
return encoded
def _make_base_file(self, checksum=True, lock=True):
"""Make a base file for testing."""
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=("$instances_path/" "%(image)s.info"), group="libvirt")
fname = os.path.join(tmpdir, "aaa")
base_file = open(fname, "w")
base_file.write("data")
base_file.close()
if lock:
lockdir = os.path.join(tmpdir, "locks")
lockname = os.path.join(lockdir, "nova-aaa")
os.mkdir(lockdir)
lock_file = open(lockname, "w")
lock_file.write("data")
lock_file.close()
base_file = open(fname, "r")
if checksum:
imagecache.write_stored_checksum(fname)
base_file.close()
yield fname
def generate_winrm_x509_cert(user_id, bits=2048):
"""Generate a cert for passwordless auth for user in project."""
subject = '/CN=%s' % user_id
upn = '%s@localhost' % user_id
with utils.tempdir() as tmpdir:
keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key'))
conffile = os.path.abspath(os.path.join(tmpdir, 'temp.conf'))
_create_x509_openssl_config(conffile, upn)
(certificate, _err) = utils.execute(
'openssl', 'req', '-x509', '-nodes', '-days', '3650',
'-config', conffile, '-newkey', 'rsa:%s' % bits,
'-outform', 'PEM', '-keyout', keyfile, '-subj', subject,
'-extensions', 'v3_req_client',
binary=True)
(out, _err) = utils.execute('openssl', 'pkcs12', '-export',
'-inkey', keyfile, '-password', 'pass:'******'ascii')
certificate = certificate.decode('utf-8')
return (private_key, certificate, fingerprint)
def _pull_missing_image(self, context, image_meta, instance):
msg = 'Image name "%s" does not exist, fetching it...'
LOG.debug(msg, image_meta['name'])
# TODO(imain): It would be nice to do this with file like object
# passing but that seems a bit complex right now.
snapshot_directory = CONF.docker.snapshots_directory
fileutils.ensure_tree(snapshot_directory)
with utils.tempdir(dir=snapshot_directory) as tmpdir:
try:
out_path = os.path.join(tmpdir, uuid.uuid4().hex)
images.fetch(context, image_meta['id'], out_path,
instance['user_id'], instance['project_id'])
self.docker.load_repository_file(
self._encode_utf8(image_meta['name']),
out_path
)
except Exception as e:
LOG.warning(_('Cannot load repository file: %s'),
e, instance=instance, exc_info=True)
msg = _('Cannot load repository file: {0}')
raise exception.NovaException(msg.format(e),
instance_id=image_meta['name'])
return self.docker.inspect_image(self._encode_utf8(image_meta['name']))
def test_concurrent_green_lock_succeeds(self):
"""Verify spawn_n greenthreads with two locks run concurrently.
This succeeds with spawn but fails with spawn_n because lockfile
gets the same thread id for both spawn_n threads. Our workaround
of using the GreenLockFile will work even if the issue is fixed.
"""
self.completed = False
with utils.tempdir() as tmpdir:
def locka(wait):
a = utils.GreenLockFile(os.path.join(tmpdir, 'a'))
a.acquire()
wait.wait()
a.release()
self.completed = True
def lockb(wait):
b = utils.GreenLockFile(os.path.join(tmpdir, 'b'))
b.acquire()
wait.wait()
b.release()
wait1 = eventlet.event.Event()
wait2 = eventlet.event.Event()
pool = greenpool.GreenPool()
pool.spawn_n(locka, wait1)
pool.spawn_n(lockb, wait2)
wait2.send()
eventlet.sleep(0)
wait1.send()
pool.waitall()
self.assertTrue(self.completed)
def _sign_csr(csr_text, ca_folder):
with utils.tempdir() as tmpdir:
inbound = os.path.join(tmpdir, 'inbound.csr')
outbound = os.path.join(tmpdir, 'outbound.csr')
try:
with open(inbound, 'w') as csrfile:
csrfile.write(csr_text)
except IOError:
LOG.exception(_('Failed to write inbound.csr'))
raise
LOG.debug(_('Flags path: %s'), ca_folder)
start = os.getcwd()
# Change working dir to CA
fileutils.ensure_tree(ca_folder)
os.chdir(ca_folder)
utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config',
'./openssl.cnf', '-infiles', inbound)
out, _err = utils.execute('openssl', 'x509', '-in', outbound,
'-serial', '-noout')
serial = string.strip(out.rpartition('=')[2])
os.chdir(start)
with open(outbound, 'r') as crtfile:
return (serial, crtfile.read())
def test_get_instance_network_qos_from_default_configure(self):
inst_type = {'vcpus': 1,
'extra_specs': {'ecus_per_vcpu:': 2}}
inst_uuid = 'fake_uuid'
qos_info = {'policy': {'key': 'ecu'},
'shaping': {'public': {'default': {'rate': 5}},
'private': {'default': {'rate': 20},
'2': {'rate': 40}}}}
default_qos = dict(private_qos=40, public_qos=5)
def fake_load(str):
return qos_info
def fake_instance_system_metadata_get(context, instance_uuid):
return {'network-qos': None}
with utils.tempdir() as d:
fname = os.path.join(d, 'test.conf')
self.flags(network_qos_config=fname)
os.system('touch %s' % fname)
self.stubs.Set(jsonutils, 'load', fake_load)
self.stubs.Set(db, 'instance_system_metadata_get',
fake_instance_system_metadata_get)
self.assertEqual(default_qos,
resource_tracker.get_instance_network_qos(inst_type,
inst_uuid))
def make_drive(self, path, image_type=IMAGE_TYPE_RAW):
"""Make the config drive.
:param path: the path to place the config drive image at
:param image_type: host side image format
:raises ProcessExecuteError if a helper process has failed.
"""
fs_format = CONF.config_drive_format
if fs_format is None:
if image_type == IMAGE_TYPE_RAW:
fs_format = FS_FORMAT_ISO9660
with utils.tempdir() as tmpdir:
self._write_md_files(tmpdir)
if image_type == IMAGE_TYPE_RAW:
if fs_format not in (FS_FORMAT_VFAT, FS_FORMAT_ISO9660):
raise exception.ConfigDriveUnsupportedFormat(
format=fs_format,
image_type=image_type,
image_path=path)
elif fs_format == FS_FORMAT_ISO9660:
self._make_iso9660(path, tmpdir)
elif fs_format == FS_FORMAT_VFAT:
self._make_vfat(path, tmpdir)
elif image_type == IMAGE_TYPE_PLOOP:
self._make_ext4_ploop(path, tmpdir)
else:
raise exception.ConfigDriveUnsupportedFormat(
format=fs_format,
image_type=image_type,
image_path=path)
def _make_base_file(self, checksum=True, lock=True):
"""Make a base file for testing."""
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'),
group='libvirt')
fname = os.path.join(tmpdir, 'aaa')
base_file = open(fname, 'w')
base_file.write('data')
base_file.close()
if lock:
lockdir = os.path.join(tmpdir, 'locks')
lockname = os.path.join(lockdir, 'nova-aaa')
os.mkdir(lockdir)
lock_file = open(lockname, 'w')
lock_file.write('data')
lock_file.close()
base_file = open(fname, 'r')
if checksum:
imagecache.write_stored_checksum(fname)
base_file.close()
yield fname
def test_concurrent_green_lock_succeeds(self):
"""Verify spawn_n greenthreads with two locks run concurrently."""
self.completed = False
with utils.tempdir() as tmpdir:
def locka(wait):
a = utils.InterProcessLock(os.path.join(tmpdir, 'a'))
with a:
wait.wait()
self.completed = True
def lockb(wait):
b = utils.InterProcessLock(os.path.join(tmpdir, 'b'))
with b:
wait.wait()
wait1 = eventlet.event.Event()
wait2 = eventlet.event.Event()
pool = greenpool.GreenPool()
pool.spawn_n(locka, wait1)
pool.spawn_n(lockb, wait2)
wait2.send()
eventlet.sleep(0)
wait1.send()
pool.waitall()
self.assertTrue(self.completed)
def test_run_image_cache_manager_pass(self):
was = {'called': False}
def fake_get_all_by_filters(context, *args, **kwargs):
was['called'] = True
instances = []
for x in range(2):
instances.append(
fake_instance.fake_db_instance(
image_ref=uuids.fake_image_ref,
uuid=getattr(uuids, 'instance_%s' % x),
name='instance-%s' % x,
vm_state='',
task_state=''))
return instances
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.stub_out('nova.db.instance_get_all_by_filters',
fake_get_all_by_filters)
compute = importutils.import_object(CONF.compute_manager)
self.flags(use_local=True, group='conductor')
compute.conductor_api = conductor.API()
ctxt = context.get_admin_context()
compute._run_image_cache_manager_pass(ctxt)
self.assertTrue(was['called'])
def test_synchronized_externally(self):
"""We can lock across multiple processes"""
with utils.tempdir() as tempdir:
self.flags(lock_path=tempdir)
rpipe1, wpipe1 = os.pipe()
rpipe2, wpipe2 = os.pipe()
@utils.synchronized('testlock1', external=True)
def f(rpipe, wpipe):
try:
os.write(wpipe, "foo")
except OSError, e:
self.assertEquals(e.errno, errno.EPIPE)
return
rfds, _wfds, _efds = select.select([rpipe], [], [], 1)
self.assertEquals(len(rfds), 0, "The other process, which was"
" supposed to be locked, "
"wrote on its end of the "
"pipe")
os.close(rpipe)
pid = os.fork()
if pid > 0:
os.close(wpipe1)
os.close(rpipe2)
f(rpipe1, wpipe2)
else:
os.close(rpipe1)
os.close(wpipe2)
f(rpipe2, wpipe1)
os._exit(0)
def test_run_image_cache_manager_pass(self, mock_instance_list):
def fake_instances(ctxt):
instances = []
for x in range(2):
instances.append(
fake_instance.fake_db_instance(
image_ref=uuids.fake_image_ref,
uuid=getattr(uuids, 'instance_%s' % x),
name='instance-%s' % x,
vm_state='',
task_state=''))
return objects.instance._make_instance_list(
ctxt, objects.InstanceList(), instances, None)
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
ctxt = context.get_admin_context()
mock_instance_list.return_value = fake_instances(ctxt)
compute = compute_manager.ComputeManager()
compute._run_image_cache_manager_pass(ctxt)
filters = {
'host': ['fake-mini'],
'deleted': False,
'soft_deleted': True,
}
mock_instance_list.assert_called_once_with(
ctxt, filters, expected_attrs=[], use_slave=True)
def test_run_image_cache_manager_pass(self):
was = {'called': False}
def fake_get_all_by_filters(context, *args, **kwargs):
was['called'] = True
instances = []
for x in range(2):
instances.append(fake_instance.fake_db_instance(
image_ref='1',
uuid=x,
name=x,
vm_state='',
task_state=''))
return instances
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.stubs.Set(db, 'instance_get_all_by_filters',
fake_get_all_by_filters)
compute = importutils.import_object(CONF.compute_manager)
self.flags(use_local=True, group='conductor')
compute.conductor_api = conductor.API()
compute._run_image_cache_manager_pass(None)
self.assertTrue(was['called'])
def test_fetch_crl_file_not_found(self):
with utils.tempdir() as tmpdir:
self.flags(ca_path=tmpdir)
self.flags(use_project_ca=True)
self.assertRaises(exception.CryptoCRLFileNotFound,
crypto.fetch_crl, project_id='fake')
def test_ensure_ca_filesystem_chdir(self, *args, **kargs):
with utils.tempdir() as tmpdir:
self.flags(ca_path=tmpdir)
start = os.getcwd()
self.assertRaises(processutils.ProcessExecutionError,
crypto.ensure_ca_filesystem)
self.assertEqual(start, os.getcwd())
def test_encrypt_decrypt_x509(self):
with utils.tempdir() as tmpdir:
self.flags(ca_path=tmpdir)
project_id = "fake"
crypto.ensure_ca_filesystem()
cert = crypto.fetch_ca(project_id)
public_key = os.path.join(tmpdir, "public.pem")
with open(public_key, 'w') as keyfile:
keyfile.write(cert)
text = "some @#!%^* test text"
process_input = text.encode("ascii") if six.PY3 else text
enc, _err = utils.execute('openssl',
'rsautl',
'-certin',
'-encrypt',
'-inkey', '%s' % public_key,
process_input=process_input,
binary=True)
dec = crypto.decrypt_text(project_id, enc)
self.assertIsInstance(dec, bytes)
if six.PY3:
dec = dec.decode('ascii')
self.assertEqual(text, dec)
def get_encoded_zip(self, project_id):
# Make a payload.zip
with utils.tempdir() as tmpdir:
filename = "payload.zip"
zippath = os.path.join(tmpdir, filename)
z = zipfile.ZipFile(zippath, "w", zipfile.ZIP_DEFLATED)
boot_script = _load_boot_script()
# genvpn, sign csr
crypto.generate_vpn_files(project_id)
z.writestr("autorun.sh", boot_script)
crl = os.path.join(crypto.ca_folder(project_id), "crl.pem")
z.write(crl, "crl.pem")
server_key = os.path.join(crypto.ca_folder(project_id), "server.key")
z.write(server_key, "server.key")
ca_crt = os.path.join(crypto.ca_path(project_id))
z.write(ca_crt, "ca.crt")
server_crt = os.path.join(crypto.ca_folder(project_id), "server.crt")
z.write(server_crt, "server.crt")
z.close()
with open(zippath, "rb") as zippy:
# NOTE(vish): run instances expects encoded userdata,
# it is decoded in the get_metadata_call.
# autorun.sh also decodes the zip file,
# hence the double encoding.
encoded = base64.b64encode(zippy.read())
encoded = base64.b64encode(encoded)
return encoded
def test_get_encoded_zip(self):
with utils.tempdir() as tmpdir:
self.flags(ca_path=tmpdir)
crypto.ensure_ca_filesystem()
ret = self.cloudpipe.get_encoded_zip(self.project)
self.assertTrue(ret)
def test_compute_manager(self):
was = {'called': False}
def fake_get_all(context, *args, **kwargs):
was['called'] = True
return [{'image_ref': '1',
'host': CONF.host,
'name': 'instance-1',
'uuid': '123',
'vm_state': '',
'task_state': ''},
{'image_ref': '1',
'host': CONF.host,
'name': 'instance-2',
'uuid': '456',
'vm_state': '',
'task_state': ''}]
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.stubs.Set(db, 'instance_get_all', fake_get_all)
compute = importutils.import_object(CONF.compute_manager)
self.flags(use_local=True, group='conductor')
compute.conductor_api = conductor.API()
compute._run_image_cache_manager_pass(None)
self.assertTrue(was['called'])
def _ssh_decrypt_text(self, ssh_private_key, text):
with utils.tempdir() as tmpdir:
sshkey = os.path.abspath(os.path.join(tmpdir, 'ssh.key'))
with open(sshkey, 'w') as f:
f.write(ssh_private_key)
try:
dec, _err = processutils.execute('openssl',
'rsautl',
'-decrypt',
'-inkey',
sshkey,
process_input=text,
binary=True)
return dec
except processutils.ProcessExecutionError as exc:
raise exception.DecryptionFailure(reason=exc.stderr)
def test_read_stored_checksum_legacy_essex(self):
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname = os.path.join(tmpdir, 'aaa')
old_fname = fname + '.sha1'
f = open(old_fname, 'w')
f.write('fdghkfhkgjjksfdgjksjkghsdf')
f.close()
csum_output = imagecache.read_stored_checksum(fname)
self.assertEquals(csum_output, 'fdghkfhkgjjksfdgjksjkghsdf')
self.assertFalse(os.path.exists(old_fname))
self.assertTrue(os.path.exists(virtutils.get_info_filename(fname)))
def test_modified_policy_reloads(self):
with utils.tempdir() as tmpdir:
tmpfilename = os.path.join(tmpdir, 'policy')
self.flags(policy_file=tmpfilename)
action = "example:test"
with open(tmpfilename, "w") as policyfile:
policyfile.write("""{"example:test": []}""")
policy.enforce(self.context, action, self.target)
with open(tmpfilename, "w") as policyfile:
policyfile.write("""{"example:test": ["false:false"]}""")
# NOTE(vish): reset stored policy cache so we don't have to
# sleep(1)
policy._POLICY_CACHE = {}
self.assertRaises(exception.PolicyNotAuthorized, policy.enforce,
self.context, action, self.target)
def test_read_stored_checksum(self):
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
csum_input = '{"sha1": "fdghkfhkgjjksfdgjksjkghsdf"}\n'
fname = os.path.join(tmpdir, 'aaa')
info_fname = virtutils.get_info_filename(fname)
f = open(info_fname, 'w')
f.write(csum_input)
f.close()
csum_output = imagecache.read_stored_checksum(fname)
self.assertEquals(csum_input.rstrip(),
'{"sha1": "%s"}' % csum_output)
def create_dir(self, host, dst_path, on_execute, on_completion):
with utils.tempdir() as tempdir:
dir_path = os.path.normpath(dst_path)
# Create target dir inside temporary directory
local_tmp_dir = os.path.join(tempdir, dir_path.strip(os.path.sep))
processutils.execute('mkdir',
'-p',
local_tmp_dir,
on_execute=on_execute,
on_completion=on_completion)
RsyncDriver._synchronize_object(tempdir,
host,
dst_path,
on_execute=on_execute,
on_completion=on_completion)
def test_configured_checksum_path(self):
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
self.flags(remove_unused_base_images=True)
# Ensure there is a base directory
os.mkdir(os.path.join(tmpdir, '_base'))
# Fake the database call which lists running instances
self.stubs.Set(
db, 'instance_get_all', lambda x: [{
'image_ref': '1',
'host': FLAGS.host,
'name': 'instance-1',
'uuid': '123',
'vm_state': '',
'task_state': ''
}, {
'image_ref': '1',
'host': FLAGS.host,
'name': 'instance-2',
'uuid': '456',
'vm_state': '',
'task_state': ''
}])
def touch(filename):
f = open(filename, 'w')
f.write('Touched')
f.close()
old = time.time() - (25 * 3600)
hashed = 'e97222e91fc4241f49a7f520d1dcf446751129b3'
base_filename = os.path.join(tmpdir, hashed)
touch(base_filename)
touch(base_filename + '.info')
os.utime(base_filename + '.info', (old, old))
touch(base_filename + '.info')
os.utime(base_filename + '.info', (old, old))
image_cache_manager = imagecache.ImageCacheManager()
image_cache_manager.verify_base_images(None)
self.assertTrue(os.path.exists(base_filename))
self.assertTrue(os.path.exists(base_filename + '.info'))
def generate_key_pair(bits=1024):
# what is the magic 65537?
with utils.tempdir() as tmpdir:
keyfile = os.path.join(tmpdir, 'temp')
utils.execute('ssh-keygen', '-q', '-b', bits, '-N', '', '-t', 'rsa',
'-f', keyfile, '-C', 'Generated by Nova')
fingerprint = _generate_fingerprint('%s.pub' % (keyfile))
if not os.path.exists(keyfile):
raise exception.FileNotFound(keyfile)
private_key = open(keyfile).read()
public_key_path = keyfile + '.pub'
if not os.path.exists(public_key_path):
raise exception.FileNotFound(public_key_path)
public_key = open(public_key_path).read()
return (private_key, public_key, fingerprint)
def test_remove_base_file_oserror(self):
with intercept_log_messages() as stream:
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
fname = os.path.join(tmpdir, 'aaa')
os.mkdir(fname)
os.utime(fname, (-1, time.time() - 3601))
# This will raise an OSError because of file permissions
image_cache_manager = imagecache.ImageCacheManager()
image_cache_manager._remove_base_file(fname)
self.assertTrue(os.path.exists(fname))
self.assertNotEqual(stream.getvalue().find('Failed to remove'),
-1)
def make_drive(self, path):
"""Make the config drive.
:param path: the path to place the config drive image at
:raises ProcessExecuteError if a helper process has failed.
"""
with utils.tempdir() as tmpdir:
self._write_md_files(tmpdir)
if CONF.config_drive_format == 'iso9660':
self._make_iso9660(path, tmpdir)
elif CONF.config_drive_format == 'vfat':
self._make_vfat(path, tmpdir)
else:
raise exception.ConfigDriveUnknownFormat(
format=CONF.config_drive_format)
def test_verify_checksum(self):
testdata = ('OpenStack Software delivers a massively scalable cloud '
'operating system.')
img = {'container_format': 'ami', 'id': '42'}
self.flags(checksum_base_images=True)
with self._intercept_log_messages() as stream:
with utils.tempdir() as tmpdir:
fname = os.path.join(tmpdir, 'aaa')
f = open(fname, 'w')
f.write(testdata)
f.close()
# Checksum is valid
f = open('%s.sha1' % fname, 'w')
csum = hashlib.sha1()
csum.update(testdata)
f.write(csum.hexdigest())
f.close()
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(img, fname)
self.assertTrue(res)
# Checksum is invalid
f = open('%s.sha1' % fname, 'w')
f.write('banana')
f.close()
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(img, fname)
self.assertFalse(res)
log = stream.getvalue()
self.assertNotEqual(log.find('image verification failed'), -1)
# Checksum file missing
os.remove('%s.sha1' % fname)
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(img, fname)
self.assertEquals(res, None)
# Checksum requests for a file with no checksum now have the
# side effect of creating the checksum
self.assertTrue(os.path.exists('%s.sha1' % fname))
def remove_dir(self, host, dst, on_execute, on_completion):
# Remove remote directory's content
with utils.tempdir() as tempdir:
processutils.execute('rsync',
'--archive',
'--delete-excluded',
tempdir + os.path.sep,
utils.format_remote_path(host, dst),
on_execute=on_execute,
on_completion=on_completion)
# Delete empty directory
RsyncDriver._remove_object(tempdir,
host,
dst,
on_execute=on_execute,
on_completion=on_completion)
def get_credentials(self, user, project=None, use_dmz=True):
"""Get credential zip for user in project"""
if not isinstance(user, User):
user = self.get_user(user)
if project is None:
project = user.id
pid = Project.safe_id(project)
private_key, signed_cert = crypto.generate_x509_cert(user.id, pid)
with utils.tempdir() as tmpdir:
zf = os.path.join(tmpdir, "temp.zip")
zippy = zipfile.ZipFile(zf, 'w')
if use_dmz and FLAGS.region_list:
regions = {}
for item in FLAGS.region_list:
region, _sep, region_host = item.partition("=")
regions[region] = region_host
else:
regions = {'nova': FLAGS.ec2_host}
for region, host in regions.iteritems():
rc = self.__generate_rc(user, pid, use_dmz, host)
zippy.writestr(FLAGS.credential_rc_file % region, rc)
zippy.writestr(FLAGS.credential_key_file, private_key)
zippy.writestr(FLAGS.credential_cert_file, signed_cert)
(vpn_ip, vpn_port) = self.get_project_vpn_data(project)
if vpn_ip:
configfile = open(FLAGS.vpn_client_template, "r")
s = string.Template(configfile.read())
configfile.close()
config = s.substitute(keyfile=FLAGS.credential_key_file,
certfile=FLAGS.credential_cert_file,
ip=vpn_ip,
port=vpn_port)
zippy.writestr(FLAGS.credential_vpn_file, config)
else:
LOG.warn(_("No vpn data for project %s"), pid)
zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(pid))
zippy.close()
with open(zf, 'rb') as f:
read_buffer = f.read()
return read_buffer
def test_encrypt_decrypt_x509(self):
with utils.tempdir() as tmpdir:
self.flags(ca_path=tmpdir)
project_id = "fake"
crypto.ensure_ca_filesystem()
cert = crypto.fetch_ca(project_id)
public_key = os.path.join(tmpdir, "public.pem")
with open(public_key, 'w') as keyfile:
keyfile.write(cert)
text = "some @#!%^* test text"
enc, _err = utils.execute('openssl',
'rsautl',
'-certin',
'-encrypt',
'-inkey', '%s' % public_key,
process_input=text)
dec = crypto.decrypt_text(project_id, enc)
self.assertEqual(text, dec)
def test_verify_checksum_file_missing(self):
self.flags(checksum_base_images=True)
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname, info_fname, testdata = self._make_checksum(tmpdir)
# Checksum file missing
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum('aaa', fname)
self.assertEquals(res, None)
# Checksum requests for a file with no checksum now have the
# side effect of creating the checksum
self.assertTrue(os.path.exists(info_fname))
def generate_key_pair(bits=None):
with utils.tempdir() as tmpdir:
keyfile = os.path.join(tmpdir, 'temp')
args = ['ssh-keygen', '-q', '-N', '', '-t', 'rsa',
'-f', keyfile, '-C', 'Generated by Nova']
if bits is not None:
args.extend(['-b', bits])
utils.execute(*args)
fingerprint = _generate_fingerprint('%s.pub' % (keyfile))
if not os.path.exists(keyfile):
raise exception.FileNotFound(keyfile)
private_key = open(keyfile).read()
public_key_path = keyfile + '.pub'
if not os.path.exists(public_key_path):
raise exception.FileNotFound(public_key_path)
public_key = open(public_key_path).read()
return (private_key, public_key, fingerprint)
def _make_base_file(self, checksum=True):
"""Make a base file for testing."""
with utils.tempdir() as tmpdir:
fname = os.path.join(tmpdir, 'aaa')
base_file = open(fname, 'w')
base_file.write('data')
base_file.close()
base_file = open(fname, 'r')
if checksum:
checksum_file = open('%s.sha1' % fname, 'w')
checksum_file.write(utils.hash_file(base_file))
checksum_file.close()
base_file.close()
yield fname
def test_handle_base_image_used_missing(self):
img = '123'
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'),
group='libvirt')
fname = os.path.join(tmpdir, 'aaa')
image_cache_manager = imagecache.ImageCacheManager()
image_cache_manager.unexplained_images = [fname]
image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])}
image_cache_manager._handle_base_image(img, fname)
self.assertEqual(image_cache_manager.unexplained_images, [])
self.assertEqual(image_cache_manager.removable_base_files, [])
def generate_x509_cert(user_id, project_id, bits=1024):
"""Generate and sign a cert for user in project."""
subject = _user_cert_subject(user_id, project_id)
with utils.tempdir() as tmpdir:
keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key'))
csrfile = os.path.join(tmpdir, 'temp.csr')
utils.execute('openssl', 'genrsa', '-out', keyfile, str(bits))
utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out',
csrfile, '-batch', '-subj', subject)
private_key = open(keyfile).read()
csr = open(csrfile).read()
(serial, signed_csr) = sign_csr(csr, project_id)
fname = os.path.join(ca_folder(project_id), 'newcerts/%s.pem' % serial)
cert = {'user_id': user_id, 'project_id': project_id, 'file_name': fname}
db.certificate_create(context.get_admin_context(), cert)
return (private_key, signed_csr)
def _pull_missing_image(self, context, image_meta, instance):
msg = 'Image name "%s" does not exist, fetching it...'
LOG.debug(msg, image_meta['name'])
shared_directory = CONF.docker.shared_directory
if (shared_directory and os.path.exists(
os.path.join(shared_directory, image_meta['id']))):
try:
self.docker.load_repository_file(
self._encode_utf8(image_meta['name']),
os.path.join(shared_directory, image_meta['id']))
return self.docker.inspect_image(
self._encode_utf8(image_meta['name']))
except Exception as e:
# If failed to load image from shared_directory, continue
# to download the image from glance then load.
LOG.warning(_('Cannot load repository file from shared '
'directory: %s'),
e,
instance=instance,
exc_info=True)
# TODO(imain): It would be nice to do this with file like object
# passing but that seems a bit complex right now.
snapshot_directory = CONF.docker.snapshots_directory
fileutils.ensure_tree(snapshot_directory)
with utils.tempdir(dir=snapshot_directory) as tmpdir:
try:
out_path = os.path.join(tmpdir, uuid.uuid4().hex)
images.fetch(context, image_meta['id'], out_path,
instance['user_id'], instance['project_id'])
self.docker.load_repository_file(
self._encode_utf8(image_meta['name']), out_path)
except Exception as e:
LOG.warning(_('Cannot load repository file: %s'),
e,
instance=instance,
exc_info=True)
msg = _('Cannot load repository file: {0}')
raise exception.NovaException(msg.format(e),
instance_id=image_meta['name'])
return self.docker.inspect_image(self._encode_utf8(image_meta['name']))
def ssh_encrypt_text(ssh_public_key, text):
"""Encrypt text with an ssh public key.
"""
with utils.tempdir() as tmpdir:
sslkey = os.path.abspath(os.path.join(tmpdir, 'ssl.key'))
try:
out = convert_from_sshrsa_to_pkcs8(ssh_public_key)
with open(sslkey, 'w') as f:
f.write(out)
enc, _err = utils.execute('openssl',
'rsautl',
'-encrypt',
'-pubin',
'-inkey', sslkey,
'-keyform', 'PEM',
process_input=text)
return enc
except processutils.ProcessExecutionError as exc:
raise exception.EncryptionFailure(reason=exc.stderr)
def _make_tgz(self, path):
try:
olddir = os.getcwd()
except OSError:
olddir = CONF.state_path
with utils.tempdir() as tmpdir:
self._write_md_files(tmpdir)
tar = tarfile.open(path, "w:gz")
os.chdir(tmpdir)
tar.add("openstack")
tar.add("ec2")
try:
os.chdir(olddir)
except Exception as e:
emsg = six.text_type(e)
LOG.debug('exception in _make_tgz %s', emsg)
tar.close()
def test_can_generate_x509(self):
with utils.tempdir() as tmpdir:
self.flags(ca_path=tmpdir)
crypto.ensure_ca_filesystem()
_key, cert_str = crypto.generate_x509_cert('fake', 'fake')
project_cert = crypto.fetch_ca(project_id='fake')
signed_cert_file = os.path.join(tmpdir, "signed")
with open(signed_cert_file, 'w') as keyfile:
keyfile.write(cert_str)
project_cert_file = os.path.join(tmpdir, "project")
with open(project_cert_file, 'w') as keyfile:
keyfile.write(project_cert)
enc, err = utils.execute('openssl', 'verify', '-CAfile',
project_cert_file, '-verbose', signed_cert_file)
self.assertFalse(err)
def _make_base_file(self, checksum=True):
"""Make a base file for testing."""
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname = os.path.join(tmpdir, 'aaa')
base_file = open(fname, 'w')
base_file.write('data')
base_file.close()
base_file = open(fname, 'r')
if checksum:
imagecache.write_stored_checksum(fname)
base_file.close()
yield fname
def test_modified_policy_reloads(self):
with utils.tempdir() as tmpdir:
tmpfilename = os.path.join(tmpdir, 'policy')
self.flags(policy_file=tmpfilename)
# NOTE(uni): context construction invokes policy check to determin
# is_admin or not. As a side-effect, policy reset is needed here
# to flush existing policy cache.
policy.reset()
action = "example:test"
with open(tmpfilename, "w") as policyfile:
policyfile.write('{"example:test": ""}')
policy.enforce(self.context, action, self.target)
with open(tmpfilename, "w") as policyfile:
policyfile.write('{"example:test": "!"}')
policy._ENFORCER.load_rules(True)
self.assertRaises(exception.PolicyNotAuthorized, policy.enforce,
self.context, action, self.target)
def test_modified_policy_reloads(self):
with utils.tempdir() as tmpdir:
tmpfilename = os.path.join(tmpdir, 'policy')
self.flags(policy_file=tmpfilename)
# NOTE(uni): context construction invokes policy check to determin
# is_admin or not. As a side-effect, policy reset is needed here
# to flush existing policy cache.
policy.reset()
action = "example:test"
with open(tmpfilename, "w") as policyfile:
policyfile.write("""{"example:test": ""}""")
policy.enforce(self.context, action, self.target)
with open(tmpfilename, "w") as policyfile:
policyfile.write("""{"example:test": "!"}""")
# NOTE(vish): reset stored policy cache so we don't have to
# sleep(1)
policy._POLICY_CACHE = {}
self.assertRaises(exception.PolicyNotAuthorized, policy.enforce,
self.context, action, self.target)
def test_verify_checksum_file_missing(self):
img = {'container_format': 'ami', 'id': '42'}
self.flags(checksum_base_images=True)
with self._intercept_log_messages() as stream:
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname, info_fname, testdata = self._make_checksum(tmpdir)
# Checksum file missing
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(img, fname)
self.assertEquals(res, None)
# Checksum requests for a file with no checksum now have the
# side effect of creating the checksum
self.assertTrue(os.path.exists(info_fname))
def _pull_missing_image(self, context, image_meta, instance):
msg = 'Image name "%s" does not exist, fetching it...'
LOG.debug(msg % image_meta['name'])
# TODO(imain): It would be nice to do this with file like object
# passing but that seems a bit complex right now.
snapshot_directory = CONF.docker.snapshots_directory
fileutils.ensure_tree(snapshot_directory)
with utils.tempdir(dir=snapshot_directory) as tmpdir:
try:
out_path = os.path.join(tmpdir, uuid.uuid4().hex)
images.fetch(context, image_meta['id'], out_path,
instance['user_id'], instance['project_id'])
self.docker.load_repository_file(image_meta['name'], out_path)
except Exception as e:
msg = _('Cannot load repository file: {0}')
raise exception.NovaException(msg.format(e),
instance_id=image_meta['name'])
return self.docker.inspect_image(image_meta['name'])
def test_verify_checksum_invalid_repaired(self):
img = {'container_format': 'ami', 'id': '42'}
self.flags(checksum_base_images=True)
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname, info_fname, testdata = self._make_checksum(tmpdir)
# Checksum is invalid, and not json
f = open(info_fname, 'w')
f.write('banana')
f.close()
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(
img, fname, create_if_missing=True)
self.assertTrue(res is None)
def _make_base_file(self, lock=True):
"""Make a base file for testing."""
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
fname = os.path.join(tmpdir, 'aaa')
base_file = open(fname, 'w')
base_file.write('data')
base_file.close()
if lock:
lockdir = os.path.join(tmpdir, 'locks')
lockname = os.path.join(lockdir, 'nova-aaa')
os.mkdir(lockdir)
lock_file = open(lockname, 'w')
lock_file.write('data')
lock_file.close()
base_file = open(fname, 'r')
base_file.close()
yield fname
def test_verify_checksum_disabled(self):
img = {'container_format': 'ami', 'id': '42'}
self.flags(checksum_base_images=False)
with self._intercept_log_messages() as stream:
with utils.tempdir() as tmpdir:
self.flags(instances_path=tmpdir)
self.flags(image_info_filename_pattern=('$instances_path/'
'%(image)s.info'))
fname, info_fname, testdata = self._make_checksum(tmpdir)
# Checksum is valid
f = open(info_fname, 'w')
csum = hashlib.sha1()
csum.update(testdata)
f.write('{"sha1": "%s"}\n' % csum.hexdigest())
f.close()
image_cache_manager = imagecache.ImageCacheManager()
res = image_cache_manager._verify_checksum(img, fname)
self.assertTrue(res is None)
