Esempio n. 1
0
    def make_resource_config(config, position, cloud_config, resource_name):
        resource_config = copy.deepcopy(cloud_config)
        resource_config[resource_name] = utils.ext_dict()
        for k, v in getattr(config,
                            '%s_%s' % (position, resource_name)).iteritems():
            resource_config[resource_name][k] = v

        return resource_config
Esempio n. 2
0
    def make_resource_config(config, position, cloud_config, resource_name):
        resource_config = copy.deepcopy(cloud_config)
        resource_config[resource_name] = utils.ext_dict()
        for k, v in getattr(config,
                            '%s_%s' % (position, resource_name)).iteritems():
            resource_config[resource_name][k] = v

        return resource_config
Esempio n. 3
0
    def make_cloud_config(config, position):
        cloud_config = utils.ext_dict(migrate=utils.ext_dict(),
                                      cloud=utils.ext_dict(),
                                      import_rules=utils.ext_dict(),
                                      mail=utils.ext_dict(),
                                      snapshot=utils.ext_dict(),
                                      mysql=utils.ext_dict())
        for k, v in config.migrate.iteritems():
            cloud_config['migrate'][k] = v

        for k, v in getattr(config, position).iteritems():
            cloud_config['cloud'][k] = v

        for k, v in config.import_rules.iteritems():
            cloud_config['import_rules'][k] = v

        for k, v in config.mail.iteritems():
            cloud_config['mail'][k] = v

        for k, v in getattr(config, position + '_mysql').iteritems():
            cloud_config['mysql'][k] = v

        cloud_config['snapshot'].update(config.snapshot)

        return cloud_config
Esempio n. 4
0
    def setUp(self):
        super(GetInfoImagesTestCase, self).setUp()

        self.fake_info = {'images': {'fake_image_id': {'image': 'image_body',
                                                       'meta': {}}}}
        self.fake_image = mock.Mock()
        self.fake_image.read_info.return_value = self.fake_info
        self.fake_src_cloud = mock.Mock()
        self.fake_dst_cloud = mock.Mock()
        self.fake_config = utils.ext_dict(migrate=utils.ext_dict(
            {'ignore_empty_images': False}))
        self.fake_src_cloud.resources = {'image': self.fake_image}

        self.fake_init = {
            'src_cloud': self.fake_src_cloud,
            'dst_cloud': self.fake_dst_cloud,
            'cfg': self.fake_config
        }
Esempio n. 5
0
    def setUp(self):
        super(ConverterVolumeToImageTest, self).setUp()
        self.fake_src_cloud = mock.Mock()
        self.fake_storage = mock.Mock()
        self.fake_storage.deploy = mock.Mock()
        self.fake_storage.upload_volume_to_image.return_value = (
            'resp', 'image_id')
        self.fake_storage.get_backend.return_value = 'ceph'
        self.fake_image = mock.Mock()
        self.fake_image.wait_for_status = mock.Mock()
        self.fake_image.read_info = mock.Mock()
        self.fake_image.read_info.return_value = {
            'images': {
                'image_id': {'image': 'image_body', 'meta': {}}}}
        self.fake_image.patch_image = mock.Mock()
        self.fake_src_cloud.resources = {'storage': self.fake_storage,
                                         'image': self.fake_image}
        self.fake_volumes_info = {
            'volumes': {
                'id1': {
                    'volume': {
                        'id': 'id1',
                        'display_name': 'dis1',

                    },
                    'meta': {
                        'image': 'image',
                    },
                }},
        }

        self.fake_dst_cloud = mock.Mock()
        self.fake_config = utils.ext_dict(migrate=utils.ext_dict(
            {'disk_format': 'qcow',
             'container_format': 'bare'}))

        self.fake_init = {
            'src_cloud': self.fake_src_cloud,
            'dst_cloud': self.fake_dst_cloud,
            'cfg': self.fake_config
        }
def _action(fake_src_data, fake_dst_data):
    fake_config = utils.ext_dict(
        migrate=utils.ext_dict({
            'ssh_connection_attempts': 3,
            'key_filename': 'key_filename',
        }),
        src=utils.ext_dict({'ssh_user': '******',
                            'ssh_sudo_password': '******',
                            'host': SRC_CINDER_HOST,
                            }),
        dst=utils.ext_dict({'ssh_user': '******',
                            'ssh_sudo_password': '******',
                            'host': DST_CINDER_HOST,
                            'conf': '/etc/cinder.conf',
                            }),
        src_storage=utils.ext_dict({'conf': '/etc/cinder.conf'}),
        dst_storage=utils.ext_dict({'conf': '/etc/cinder.conf'}),
    )

    fake_src_cloud = mock.Mock()
    fake_src_storage = mock.Mock()
    fake_src_cloud.resources = {'storage': fake_src_storage}

    fake_dst_cloud = mock.Mock()
    fake_dst_storage = mock.Mock()
    fake_dst_storage.read_db_info = \
        mock.Mock(return_value=jsondate.dumps(fake_dst_data))
    fake_dst_cloud.resources = {'storage': fake_dst_storage}

    fake_init = {
        'src_cloud': fake_src_cloud,
        'dst_cloud': fake_dst_cloud,
        'cfg': fake_config
    }

    action = cinder_database_manipulation.WriteVolumesDb(fake_init)

    action.dst_mount = get_dst_mount(fake_dst_data)

    action.mount_dirs = mock.MagicMock(side_effect=mount_dirs)

    action.find_dir = mock.MagicMock(side_effect=find_dir(fake_dst_data))

    action.volume_size = mock.MagicMock(side_effect=volume_size)

    action.free_space = mock.MagicMock(side_effect=free_space)

    action.dst_hosts = [
        'dst_cinder',
        'dst_cinder@nfs1',
        'dst_cinder@nfs2',
        'dst_cinder@nfs3',
    ]
    action.run_repeat_on_errors = mock.Mock()

    args = {
        cinder_database_manipulation.NAMESPACE_CINDER_CONST:
        jsondate.dumps(fake_src_data)
    }
    return action, args
Esempio n. 7
0
    def setUp(self):
        super(CopyFromGlanceToGlanceTestCase, self).setUp()

        self.fake_input_info = {'image_data': {'fake_key': 'fake_value'}}
        self.fake_result_info = {'image_data': {
            'image': {'images': [{'image': 'image_body', 'meta': {}}]}}}

        self.fake_image = mock.Mock()
        self.fake_image.deploy.return_value = self.fake_result_info
        self.src_cloud = mock.Mock()
        self.dst_cloud = mock.Mock()
        self.dst_cloud.resources = {'image': self.fake_image}

        self.fake_config = utils.ext_dict(migrate=utils.ext_dict(
            {'ignore_empty_images': False}))
        self.src_cloud.resources = {'image': self.fake_image}

        self.fake_init = {
            'src_cloud': self.src_cloud,
            'dst_cloud': self.dst_cloud,
            'cfg': self.fake_config
        }
Esempio n. 8
0
    def make_cloud_config(config, position):
        cloud_config = utils.ext_dict(migrate=utils.ext_dict(),
                                      cloud=utils.ext_dict(),
                                      import_rules=utils.ext_dict(),
                                      mail=utils.ext_dict(),
                                      snapshot=utils.ext_dict(),
                                      mysql=utils.ext_dict(),
                                      rabbit=utils.ext_dict(),
                                      storage=utils.ext_dict(),
                                      initial_check=utils.ext_dict())

        cloud_config['migrate'].update(config.migrate)
        cloud_config['cloud'].update(getattr(config, position))
        cloud_config['import_rules'].update(config.import_rules)
        cloud_config['mail'].update(config.mail)
        cloud_config['mysql'].update(getattr(config, position + '_mysql'))
        cloud_config['rabbit'].update(getattr(config, position + '_rabbit'))
        cloud_config['snapshot'].update(config.snapshot)
        cloud_config['storage'].update(getattr(config, position + '_storage'))
        cloud_config['initial_check'].update(config.initial_check)

        return cloud_config
Esempio n. 9
0
    def make_cloud_config(config, position):
        cloud_config = utils.ext_dict(migrate=utils.ext_dict(),
                                      cloud=utils.ext_dict(),
                                      import_rules=utils.ext_dict(),
                                      mail=utils.ext_dict(),
                                      snapshot=utils.ext_dict(),
                                      mysql=utils.ext_dict(),
                                      rabbit=utils.ext_dict(),
                                      storage=utils.ext_dict(),
                                      initial_check=utils.ext_dict())

        cloud_config['migrate'].update(config.migrate)
        cloud_config['cloud'].update(getattr(config, position))
        cloud_config['import_rules'].update(config.import_rules)
        cloud_config['mail'].update(config.mail)
        cloud_config['mysql'].update(getattr(config, position + '_mysql'))
        cloud_config['rabbit'].update(getattr(config, position + '_rabbit'))
        cloud_config['snapshot'].update(config.snapshot)
        cloud_config['storage'].update(getattr(config, position + '_storage'))
        cloud_config['initial_check'].update(config.initial_check)

        return cloud_config
Esempio n. 10
0
    def make_cloud_config(config, position):
        cloud_config = utils.ext_dict(migrate=utils.ext_dict(),
                                      cloud=utils.ext_dict(),
                                      import_rules=utils.ext_dict(),
                                      mail=utils.ext_dict(),
                                      mysql=utils.ext_dict())
        for k, v in config.migrate.iteritems():
            cloud_config['migrate'][k] = v

        for k, v in getattr(config, position).iteritems():
            cloud_config['cloud'][k] = v

        for k, v in config.import_rules.iteritems():
            cloud_config['import_rules'][k] = v

        for k, v in config.mail.iteritems():
            cloud_config['mail'][k] = v

        for k, v in getattr(config, position + '_mysql').iteritems():
            cloud_config['mysql'][k] = v

        return cloud_config
Esempio n. 11
0
from cloudferrylib.utils import utils
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'auth_url': 'http://1.1.1.1:35357/v2.0/',
    'region': None,
    'service_tenant': 'service'
}),
                             migrate=utils.ext_dict({
                                 'speed_limit':
                                 '10MB',
                                 'retry':
                                 '7',
                                 'time_wait':
                                 5,
                                 'keep_user_passwords':
                                 False,
                                 'overwrite_user_passwords':
                                 False,
                                 'migrate_users':
                                 True,
                                 'optimize_user_role_fetch':
                                 False
                             }),
                             mail=utils.ext_dict({'server': '-'}))


@mock.patch("cloudferrylib.base.clients", mock.MagicMock())
Esempio n. 12
0
from cinderclient.v1 import client as cinder_client
from oslotest import mockpatch

from cloudferrylib.os.storage import cinder_storage
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'host': '1.1.1.1'}),
    migrate=utils.ext_dict({'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': '5',
                            'keep_volume_storage': False}),
    mysql=utils.ext_dict({'host': '1.1.1.1'}),
    storage=utils.ext_dict({'backend': 'ceph',
                            'rbd_pool': 'volumes',
                            'volume_name_template': 'volume-',
                            'host': '1.1.1.1'}))


class CinderStorageTestCase(test.TestCase):
    def setUp(self):
        super(CinderStorageTestCase, self).setUp()
        self.mock_client = mock.Mock()
        self.cs_patch = mockpatch.PatchObject(cinder_client, 'Client',
                                              new=self.mock_client)
        self.useFixture(self.cs_patch)
Esempio n. 13
0
import mock

from oslotest import mockpatch

from cloud import cloud
from cloud import grouping
from cloudferrylib.utils import utils
from tests import test


RESULT_FILE = 'tests/grouping_result'
FILE_NAME = 'tests/groups'
FAKE_CONFIG = utils.ext_dict(src=utils.ext_dict({'user': '******',
                                                 'password': '******',
                                                 'tenant': 'fake_tenant',
                                                 'host': '1.1.1.1'}),
                             migrate=utils.ext_dict(
                                 {'group_file_path': RESULT_FILE}))


class GroupingTestCase(test.TestCase):
    def setUp(self):
        super(GroupingTestCase, self).setUp()

        self.network = mock.Mock()
        self.compute = mock.Mock()
        self.identity = mock.Mock()

        self.fake_tenant1 = mock.Mock()
        self.fake_tenant1.id = 't1'
        self.fake_tenant2 = mock.Mock()
Esempio n. 14
0
from oslotest import mockpatch

from cloudferrylib.base import exception
from cloudferrylib.os.compute import nova_compute
from cloudferrylib.utils import utils

from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'region': None,
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'cacert': '',
                          'insecure': False}),
    mysql=utils.ext_dict({'host': '1.1.1.1'}),
    migrate=utils.ext_dict({'migrate_quotas': True,
                            'retry': '7',
                            'time_wait': 5,
                            'keep_network_interfaces_order': True,
                            'keep_usage_quotas_inst': True}))


class BaseNovaComputeTestCase(test.TestCase):
    def setUp(self):
        super(BaseNovaComputeTestCase, self).setUp()

        self.mock_client = mock.MagicMock()
        self.nc_patch = mockpatch.PatchObject(nova_client, 'Client',
                                              new=self.mock_client)
Esempio n. 15
0
import copy
import mock

from neutronclient.v2_0 import client as neutron_client
from oslotest import mockpatch

from cloudferrylib.os.network import neutron
from cloudferrylib.utils import utils
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'host': '1.1.1.1',
}),
                             migrate=utils.ext_dict({
                                 'speed_limit': '10MB',
                                 'retry': '7',
                                 'time_wait': '5'
                             }))


class NeutronTestCase(test.TestCase):
    def setUp(self):
        super(NeutronTestCase, self).setUp()

        self.neutron_mock_client = mock.MagicMock()

        self.neutron_client_patch = \
            mockpatch.PatchObject(neutron_client,
def _action(fake_src_data, fake_dst_data, fake_deployed_data):
    fake_config = utils.ext_dict(
        migrate=utils.ext_dict({
            'ssh_connection_attempts': 3,
            'key_filename': 'key_filename',
        }),
        src=utils.ext_dict({'ssh_user': '******',
                            'ssh_sudo_password': '******',
                            'host': SRC_CINDER_HOST,
                            }),
        dst=utils.ext_dict({'ssh_user': '******',
                            'ssh_sudo_password': '******',
                            'host': DST_CINDER_HOST,
                            'conf': '/etc/cinder.conf',
                            }),
        src_storage=utils.ext_dict({'conf': '/etc/cinder.conf'}),
        dst_storage=utils.ext_dict({'conf': '/etc/cinder.conf'}),
    )

    fake_src_cloud = mock.Mock()
    fake_src_storage = mock.Mock()
    fake_src_storage.read_db_info = \
        mock.Mock(return_value=fake_src_data)
    fake_img_res = mock.Mock()

    fake_src_cloud.migration = {
        'image': FakeMigration('image'),
        'identity': None,
    }
    fake_src_cloud.resources = {
        'storage': fake_src_storage,
        'image': fake_img_res,
    }
    fake_src_images = {
        'images':
        {
            'img1': {
                'image': {
                    'id': 'img1',
                    'name': 'img1_name',
                    'checksum': 'fake_checksum1',
                }
            }
        }
    }
    fake_img_res.read_db_info = \
        mock.Mock(return_value=fake_src_images)

    fake_dst_cloud = mock.Mock()
    fake_dst_storage = mock.Mock()
    fake_dst_storage.read_db_info = \
        mock.Mock(return_value=fake_dst_data)
    fake_dst_storage.reread = \
        mock.Mock(return_value=fake_deployed_data)

    fake_dst_storage.deploy = mock.Mock(side_effect=no_modify)
    fake_dst_img_res = mock.Mock()
    fake_dst_cloud.resources = {
        'storage': fake_dst_storage,
        'image': fake_dst_img_res,
    }
    fake_dst_images = {
        'images':
        {
            'dst_img1': {
                'image': {
                    'id': 'dst_img1',
                    'name': 'img1_name',
                    'checksum': 'fake_checksum1',
                }
            }
        }
    }
    fake_dst_img_res.read_db_info = \
        mock.Mock(return_value=fake_dst_images)

    fake_init = {
        'src_cloud': fake_src_cloud,
        'dst_cloud': fake_dst_cloud,
        'cfg': fake_config
    }

    action = cinder_database_manipulation.WriteVolumesDb(fake_init)

    action.cp_volumes.dst_mount = get_dst_mount(fake_dst_data)

    action.cp_volumes.mount_dirs = mock.MagicMock(side_effect=mount_dirs)

    action.cp_volumes.find_dir = mock.MagicMock(
        side_effect=find_dir(fake_dst_data))

    action.cp_volumes.volume_size = mock.MagicMock(side_effect=volume_size)

    action.cp_volumes.free_space = mock.MagicMock(side_effect=free_space)

    action.cp_volumes.dst_volumes = mock.MagicMock(return_value=[])

    action.cp_volumes.dst_hosts = [
        'dst_cinder',
        'dst_cinder@nfs1',
        'dst_cinder@nfs2',
        'dst_cinder@nfs3',
    ]

    action.cp_volumes.run_repeat_on_errors = mock.Mock()

    def not_rsync(_, src, dst):
        return action.cp_volumes.run_rsync(src, dst)
    action.cp_volumes.rsync_if_enough_space = \
        mock.MagicMock(side_effect=not_rsync)

    return action, {
        cinder_database_manipulation.NAMESPACE_CINDER_CONST:
        fake_src_data
    }
Esempio n. 17
0
import mock

from glanceclient.v1 import client as glance_client
from oslotest import mockpatch

from cloudferrylib.os.image.glance_image import GlanceImage
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({'user': '******',
                                                   'password': '******',
                                                   'tenant': 'fake_tenant',
                                                   'region': None,
                                                   'host': '1.1.1.1',
                                                   'ssh_host': '1.1.1.10',
                                                   'ssh_user': '******',
                                                   'cacert': '',
                                                   'insecure': False
                                                   }),
                             migrate=utils.ext_dict({'retry': '7',
                                                     'time_wait': 5}))


class FakeUser(object):
    def __init__(self):
        self.name = 'fake_user_name'


class GlanceImageTestCase(test.TestCase):

    def setUp(self):
Esempio n. 18
0

def tn_name_by_id(uuid, _):
    for tn in TENANTS:
        if tn['id'] == uuid:
            return tn['name']

FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'host': '1.1.1.1',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'cacert': '',
                          'insecure': False}),
    migrate=utils.ext_dict({'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': 5,
                            'keep_volume_storage': False,
                            'keep_volume_snapshots': False}),
    mysql=utils.ext_dict({'db_host': '1.1.1.1'}),
    storage=utils.ext_dict({'backend': 'ceph',
                            'rbd_pool': 'volumes',
                            'volume_name_template': 'volume-',
                            'host': '1.1.1.1'}))


STATUSES = (
    AVAILABLE, IN_USE, CREATING, ERROR, DELETING, ERROR_DELETING,
    ATTACHING, DETACHING, ERROR_ATTACHING,
) = (
    'available', 'in-use', 'creating', 'error', 'deleting', 'error_deleting',
Esempio n. 19
0
from cloudferrylib.utils import utils

from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'region': None,
    'auth_url': 'http://1.1.1.1:35357/v2.0/',
    'cacert': '',
    'insecure': False
}),
                             mysql=utils.ext_dict({'host': '1.1.1.1'}),
                             migrate=utils.ext_dict({
                                 'migrate_quotas':
                                 True,
                                 'retry':
                                 '7',
                                 'time_wait':
                                 5,
                                 'keep_network_interfaces_order':
                                 True,
                                 'keep_usage_quotas_inst':
                                 True,
                                 'override_rules':
                                 None
                             }))


class BaseNovaComputeTestCase(test.TestCase):
    def setUp(self):
Esempio n. 20
0
import mock

from novaclient.v1_1 import client as nova_client
from oslotest import mockpatch

from cloudferrylib.os.network.nova_network import NovaNetwork
from cloudferrylib.utils import utils
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'region': None,
    'auth_url': 'http://1.1.1.1:35357/v2.0/',
    'cacert': '',
    'insecure': False
}),
                             migrate=utils.ext_dict({
                                 'retry': '7',
                                 'time_wait': 5
                             }))


class TestNovaNetwork(test.TestCase):
    def setUp(self):
        super(TestNovaNetwork, self).setUp()

        self.nova_mock_client = mock.MagicMock()

        self.nova_client_patch = mockpatch.PatchObject(
Esempio n. 21
0
from oslotest import mockpatch

from cloudferrylib.base import exception
from cloudferrylib.os.network import neutron
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'region': None,
                          'service_tenant': 'services',
                          'cacert': '',
                          'insecure': False}),
    migrate=utils.ext_dict({'ext_net_map': 'fake_ext_net_map.yaml',
                            'retry': '7',
                            'time_wait': 5}),
    network=utils.ext_dict({
        'get_all_quota': True
    }))


class NeutronTestCase(test.TestCase):

    def setUp(self):
        super(NeutronTestCase, self).setUp()

        self.neutron_mock_client = mock.MagicMock()
Esempio n. 22
0
    def deploy(self, info):
        LOG.info("Glance images deployment started...")
        info = copy.deepcopy(info)
        new_info = {'images': {}}
        created_images = []
        delete_container_format, delete_disk_format = [], []
        empty_image_list = {}

        # List for obsolete/broken images IDs, that will not be migrated
        obsolete_images_ids_list = []
        dst_img_checksums = {x.checksum: x for x in self.get_image_list()}
        dst_img_names = [x.name for x in self.get_image_list()]

        for image_id_src, gl_image in info['images'].iteritems():
            img = gl_image['image']
            if img and img['resource']:
                checksum_current = img['checksum']
                name_current = img['name']
                meta = gl_image['meta']
                same_image_on_destination = (
                    checksum_current in dst_img_checksums and
                    name_current in dst_img_names)

                if same_image_on_destination:
                    created_images.append(
                        (dst_img_checksums[checksum_current], meta))
                    LOG.info("Image '%s' is already present on destination, "
                             "skipping", img['name'])
                    continue

                LOG.debug("Updating owner '{owner}' of image '{image}'".format(
                    owner=img["owner_name"],
                    image=img["name"]))
                img["owner"] = \
                    self.identity_client.get_tenant_id_by_name(
                    img["owner_name"])
                del img["owner_name"]

                if img["properties"]:
                    # update snapshot metadata
                    metadata = img["properties"]
                    if "owner_id" in metadata:
                        # update tenant id
                        LOG.debug("updating snapshot metadata for field "
                                  "'owner_id' for image {image}".format(
                                      image=img["id"]))
                        metadata["owner_id"] = img["owner"]
                    if "user_id" in metadata:
                        # update user id by specified name
                        LOG.debug("updating snapshot metadata for field "
                                  "'user_id' for image {image}".format(
                                      image=img["id"]))
                        try:
                            ks_client = self.identity_client.keystone_client
                            metadata["user_id"] = ks_client.users.find(
                                username=metadata["user_name"]).id
                            del metadata["user_name"]
                        except keystone_exceptions.NotFound:
                            LOG.warning("Cannot update user name for image "
                                        "{}".format(img['name']))
                if img["checksum"] is None:
                    LOG.warning("re-creating image {} "
                                "from original source URL"
                                .format(img["id"]))
                    if meta['img_loc'] is not None:
                        self.glance_img_create(
                            img['name'],
                            img['disk_format'] or "qcow2",
                            meta['img_loc']
                        )
                        recreated_image = utl.ext_dict(
                            name=img["name"]
                        )
                        created_images.append(
                            (recreated_image, gl_image['meta'])
                        )
                    else:
                        raise exception.AbortMigrationError(
                            "image information has no original source URL")
                    continue

                LOG.debug("Creating image '{image}' ({image_id})".format(
                    image=img["name"],
                    image_id=img['id']))
                # we can face situation when image has no
                # disk_format and container_format properties
                # this situation appears, when image was created
                # with option --copy-from
                # glance-client cannot create image without this
                # properties, we need to create them artificially
                # and then - delete from database

                try:
                    data_proxy = file_like_proxy.FileLikeProxy(
                        img, self.config['migrate']['speed_limit'])

                    created_image = self.create_image(
                        name=img['name'],
                        container_format=(img['container_format'] or "bare"),
                        disk_format=(img['disk_format'] or "qcow2"),
                        is_public=img['is_public'],
                        protected=img['protected'],
                        owner=img['owner'],
                        size=img['size'],
                        properties=img['properties'],
                        data=data_proxy)

                    image_members = img['members'].get(img['id'], {})
                    for tenant_name, can_share in image_members.iteritems():
                        LOG.debug("deploying image member for image '%s' "
                                  "tenant '%s'", img['id'], img['owner'])
                        self.create_member(
                            created_image.id, tenant_name, can_share)

                    LOG.debug("new image ID {}".format(created_image.id))
                    created_images.append((created_image, meta))
                except exception.ImageDownloadError:
                    LOG.warning("Unable to reach image's data due to "
                                "Glance HTTPInternalServerError. Skipping "
                                "image: (id = %s)", img["id"])
                    obsolete_images_ids_list.append(img["id"])
                    continue

                if not img["container_format"]:
                    delete_container_format.append(created_image.id)
                if not img["disk_format"]:
                    delete_disk_format.append(created_image.id)
            elif img['resource'] is None:
                recreated_image = utl.ext_dict(name=img["name"])
                created_images.append((recreated_image, gl_image['meta']))
            elif not img:
                empty_image_list[image_id_src] = gl_image

        # Remove obsolete/broken images from info
        for img_id in obsolete_images_ids_list:
            info['images'].pop(img_id)

        if created_images:
            im_name_list = [(im.name, tmp_meta) for (im, tmp_meta) in
                            created_images]
            LOG.debug("images on destination: {}".format(
                [im for (im, tmp_meta) in im_name_list]))
            new_info = self._convert_images_with_metadata(im_name_list)
        new_info['images'].update(empty_image_list)
        self.delete_fields('disk_format', delete_disk_format)
        self.delete_fields('container_format', delete_container_format)
        LOG.info("Glance images deployment finished.")
        return new_info
Esempio n. 23
0
from cloudferrylib.os.storage import filters
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict(
        {
            "user": "******",
            "password": "******",
            "tenant": "fake_tenant",
            "host": "1.1.1.1",
            "ssh_host": "1.1.1.10",
            "auth_url": "http://1.1.1.1:35357/v2.0/",
            "region": None,
            "cacert": "",
            "insecure": False,
        }
    ),
    migrate=utils.ext_dict(
        {"retry": "7", "time_wait": 5, "keep_volume_storage": False, "keep_volume_snapshots": False}
    ),
    mysql=utils.ext_dict({"db_host": "1.1.1.1"}),
    storage=utils.ext_dict(
        {"backend": "ceph", "rbd_pool": "volumes", "volume_name_template": "volume-", "host": "1.1.1.1"}
    ),
)


class CinderStorageTestCase(test.TestCase):
    def setUp(self):
        test.TestCase.setUp(self)
Esempio n. 24
0
from cloudferrylib.os.compute import nova_compute
from cloudferrylib.utils import timeout_exception
from cloudferrylib.utils import utils

from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict(
        {
            "user": "******",
            "password": "******",
            "tenant": "fake_tenant",
            "region": None,
            "auth_url": "http://1.1.1.1:35357/v2.0/",
            "cacert": "",
            "insecure": False,
        }
    ),
    mysql=utils.ext_dict({"host": "1.1.1.1"}),
    migrate=utils.ext_dict({"migrate_quotas": True, "retry": "7", "time_wait": 5}),
)


class NovaComputeTestCase(test.TestCase):
    def setUp(self):
        super(NovaComputeTestCase, self).setUp()

        self.mock_client = mock.MagicMock()
        self.nc_patch = mockpatch.PatchObject(nova_client, "Client", new=self.mock_client)
Esempio n. 25
0
    def deploy(self, info):
        LOG.info("Glance images deployment started...")
        info = copy.deepcopy(info)
        new_info = {'images': {}}
        migrate_images_list = []
        delete_container_format, delete_disk_format = [], []
        empty_image_list = {}

        # List for obsolete/broken images IDs, that will not be migrated
        obsolete_images_ids_list = []

        for image_id_src, gl_image in info['images'].iteritems():
            if gl_image['image'] and gl_image['image']['resource']:
                dst_img_checksums = {x.checksum: x for x in
                                     self.get_image_list()}
                dst_img_names = [x.name for x in self.get_image_list()]
                checksum_current = gl_image['image']['checksum']
                name_current = gl_image['image']['name']
                meta = gl_image['meta']
                if checksum_current in dst_img_checksums and (
                        name_current) in dst_img_names:
                    migrate_images_list.append(
                        (dst_img_checksums[checksum_current], meta))
                    continue

                LOG.debug("Updating owner '{owner}' of image '{image}'".format(
                    owner=gl_image["image"]["owner_name"],
                    image=gl_image["image"]["name"]))
                gl_image["image"]["owner"] = \
                    self.identity_client.get_tenant_id_by_name(
                    gl_image["image"]["owner_name"])
                del gl_image["image"]["owner_name"]

                if gl_image["image"]["properties"]:
                    # update snapshot metadata
                    metadata = gl_image["image"]["properties"]
                    if "owner_id" in metadata:
                        # update tenant id
                        LOG.debug("updating snapshot metadata for field "
                                  "'owner_id' for image {image}".format(
                                      image=gl_image["image"]["id"]))
                        metadata["owner_id"] = gl_image["image"]["owner"]
                    if "user_id" in metadata:
                        # update user id by specified name
                        LOG.debug("updating snapshot metadata for field "
                                  "'user_id' for image {image}".format(
                                      image=gl_image["image"]["id"]))
                        metadata["user_id"] = \
                            self.identity_client.keystone_client.users.find(
                                username=metadata["user_name"]).id
                        del metadata["user_name"]
                if gl_image["image"]["checksum"] is None:
                    LOG.warning("re-creating image {} "
                                "from original source URL"
                                .format(gl_image["image"]["id"]))
                    if meta['img_loc'] is not None:
                        self.glance_img_create(
                            gl_image['image']['name'],
                            gl_image['image']['disk_format'] or "qcow2",
                            meta['img_loc']
                        )
                        recreated_image = utl.ext_dict(
                            name=gl_image["image"]["name"]
                        )
                        migrate_images_list.append(
                            (recreated_image, gl_image['meta'])
                        )
                    else:
                        raise exception.AbortMigrationError(
                            "image information has no original source URL")
                    continue

                LOG.debug("Creating image '{image}' ({image_id})".format(
                    image=gl_image["image"]["name"],
                    image_id=gl_image['image']['id']))
                # we can face situation when image has no
                # disk_format and container_format properties
                # this situation appears, when image was created
                # with option --copy-from
                # glance-client cannot create image without this
                # properties, we need to create them artificially
                # and then - delete from database

                try:
                    migrate_image = self.create_image(
                        name=gl_image['image']['name'],
                        container_format=(gl_image['image']['container_format']
                                          or "bare"),
                        disk_format=(gl_image['image']['disk_format']
                                     or "qcow2"),
                        is_public=gl_image['image']['is_public'],
                        protected=gl_image['image']['protected'],
                        owner=gl_image['image']['owner'],
                        size=gl_image['image']['size'],
                        properties=gl_image['image']['properties'],
                        data=file_like_proxy.FileLikeProxy(
                            gl_image['image'],
                            self.config['migrate']['speed_limit']))
                    LOG.debug("new image ID {}".format(migrate_image.id))
                except exception.ImageDownloadError:
                    LOG.warning("Unable to reach image's data due to "
                                "Glance HTTPInternalServerError. Skipping "
                                "image: (id = %s)", gl_image["image"]["id"])
                    obsolete_images_ids_list.append(gl_image["image"]["id"])
                    continue

                migrate_images_list.append((migrate_image, meta))
                if not gl_image["image"]["container_format"]:
                    delete_container_format.append(migrate_image.id)
                if not gl_image["image"]["disk_format"]:
                    delete_disk_format.append(migrate_image.id)
            elif gl_image['image']['resource'] is None:
                recreated_image = utl.ext_dict(name=gl_image["image"]["name"])
                migrate_images_list.append((recreated_image, gl_image['meta']))
            elif not gl_image['image']:
                empty_image_list[image_id_src] = gl_image

        # Remove obsolete/broken images from info
        [info['images'].pop(img_id) for img_id in obsolete_images_ids_list]

        if migrate_images_list:
            im_name_list = [(im.name, tmp_meta) for (im, tmp_meta) in
                            migrate_images_list]
            LOG.debug("images on destination: {}".format(
                [im for (im, tmp_meta) in im_name_list]))
            new_info = self.read_info(images_list_meta=im_name_list)
        new_info['images'].update(empty_image_list)
        # on this step we need to create map between source ids and dst ones
        LOG.debug("creating map between source and destination image ids")
        image_ids_map = {}
        dst_img_checksums = {x.checksum: x.id for x in self.get_image_list()}
        for image_id_src, gl_image in info['images'].iteritems():
            cur_image = gl_image["image"]
            image_ids_map[cur_image["id"]] = \
                dst_img_checksums[cur_image["checksum"]]
        LOG.debug("deploying image members")
        for image_id, data in info.get("members", {}).items():
            for tenant_name, can_share in data.items():
                LOG.debug("deploying image member for image {image}"
                          " tenant {tenant}".format(
                              image=image_id,
                              tenant=tenant_name))
                self.create_member(
                    image_ids_map[image_id],
                    tenant_name,
                    can_share)
        self.delete_fields('disk_format', delete_disk_format)
        self.delete_fields('container_format', delete_container_format)
        LOG.info("Glance images deployment finished.")
        return new_info
import copy

import mock

from cloudferrylib.os.storage import cinder_database
from cloudferrylib.os.storage import cinder_netapp
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'host': '1.1.1.1',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/'}),
    mysql=utils.ext_dict({'db_host': '1.1.1.1'}),
    migrate=utils.ext_dict({
        'retry': '7',
        'time_wait': 5}))

FAKE_ENTRY_0 = {'id': 'fake_volume_id_0',
                'host': 'c1',
                'provider_location': 'fake_netapp_server_0:/vol/v00_cinder'}

FAKE_ENTRY_1 = {'id': 'fake_volume_id_1',
                'host': 'c2@another_wrong_id',
                'provider_location': 'fake_netapp_server_1:/vol/v01_cinder'}


class CinderNetAppTestCase(test.TestCase):
Esempio n. 27
0
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'host': '1.1.1.1',
    'ssh_host': '1.1.1.10',
    'auth_url': 'http://1.1.1.1:35357/v2.0/',
    'region': None,
    'cacert': '',
    'insecure': False
}),
                             migrate=utils.ext_dict({
                                 'retry':
                                 '7',
                                 'time_wait':
                                 5,
                                 'keep_volume_storage':
                                 False,
                                 'keep_volume_snapshots':
                                 False
                             }),
                             mysql=utils.ext_dict({'db_host': '1.1.1.1'}),
                             storage=utils.ext_dict({
                                 'backend': 'ceph',
                                 'rbd_pool': 'volumes',
                                 'volume_name_template': 'volume-',
                                 'host': '1.1.1.1'
                             }))

Esempio n. 28
0

import mock

from novaclient.v1_1 import client as nova_client
from oslotest import mockpatch

from cloudferrylib.os.network.nova_network import NovaNetwork
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/'}),
    migrate=utils.ext_dict({'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': 5}))


class TestNovaNetwork(test.TestCase):
    def setUp(self):
        super(TestNovaNetwork, self).setUp()

        self.nova_mock_client = mock.MagicMock()

        self.nova_client_patch = mockpatch.PatchObject(
            nova_client,
            'Client',
            new=self.nova_mock_client)
Esempio n. 29
0
from glanceclient.v1 import client as glance_client
from oslotest import mockpatch

from cloudferrylib.os.image.glance_image import GlanceImage
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict(
        {
            "user": "******",
            "password": "******",
            "tenant": "fake_tenant",
            "region": None,
            "host": "1.1.1.1",
            "ssh_user": "******",
            "cacert": "",
            "insecure": False,
        }
    ),
    migrate=utils.ext_dict({"retry": "7", "time_wait": 5}),
)


class FakeUser(object):
    def __init__(self):
        self.name = "fake_user_name"


class GlanceImageTestCase(test.TestCase):
Esempio n. 30
0
import mock

from neutronclient.v2_0 import client as neutron_client
from oslotest import mockpatch

from cloudferrylib.os.network import neutron
from cloudferrylib.utils import utils
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'auth_url': 'http://1.1.1.1:35357/v2.0/',
    'region': None,
    'service_tenant': 'services'
}),
                             migrate=utils.ext_dict({
                                 'ext_net_map': 'fake_ext_net_map.yaml',
                                 'speed_limit': '10MB',
                                 'retry': '7',
                                 'time_wait': 5
                             }))


class NeutronTestCase(test.TestCase):
    def setUp(self):
        super(NeutronTestCase, self).setUp()

        self.neutron_mock_client = mock.MagicMock()

        self.neutron_client_patch = \
Esempio n. 31
0
    def deploy(self, info):
        LOG.info("Glance images deployment started...")
        info = copy.deepcopy(info)
        new_info = {'images': {}}
        created_images = []
        delete_container_format, delete_disk_format = [], []
        empty_image_list = {}
        keystone = self.cloud.resources["identity"]

        # List for obsolete/broken images IDs, that will not be migrated
        obsolete_images_ids_list = []
        dst_images = {}
        for dst_image in self.get_image_list():
            tenant_name = keystone.try_get_tenant_name_by_id(
                dst_image.owner, default=self.cloud.cloud_config.cloud.tenant)
            image_key = (dst_image.name, tenant_name, dst_image.checksum,
                         dst_image.is_public)
            dst_images[image_key] = dst_image

        view = GlanceImageProgessMigrationView(info['images'], dst_images)
        view.show_info()
        for image_id_src in info['images']:
            img = info['images'][image_id_src]['image']
            meta = info['images'][image_id_src]['meta']
            if img and img['resource']:
                checksum_current = img['checksum']
                name_current = img['name']
                tenant_name = img['owner_name']
                image_key = (name_current, tenant_name, checksum_current,
                             img['is_public'])

                if image_key in dst_images:
                    existing_image = dst_images[image_key]
                    created_images.append((existing_image, meta))
                    image_members = img['members'].get(img['id'], {})
                    self.update_membership(existing_image.id, image_members)
                    LOG.info("Image '%s' is already present on destination, "
                             "skipping", img['name'])
                    continue

                view.show_progress()
                view.inc_progress(img['size'])

                LOG.debug("Updating owner '%s' of image '%s'",
                          tenant_name, img["name"])
                img["owner"] = \
                    self.identity_client.get_tenant_id_by_name(tenant_name)

                if img["properties"]:
                    # update snapshot metadata
                    metadata = img["properties"]
                    if "owner_id" in metadata:
                        # update tenant id
                        LOG.debug("Updating snapshot metadata for field "
                                  "'owner_id' for image %s", img["id"])
                        metadata["owner_id"] = img["owner"]
                    if "user_name" in metadata:
                        # update user id by specified name
                        LOG.debug("Updating snapshot metadata for field "
                                  "'user_id' for image %s", img["id"])
                        try:
                            ks_client = self.identity_client.keystone_client
                            metadata["user_id"] = ks_client.users.find(
                                username=metadata["user_name"]).id
                            del metadata["user_name"]
                        except keystone_exceptions.NotFound:
                            LOG.warning("Cannot update user name for image %s",
                                        img['name'])
                if img["checksum"] is None:
                    LOG.warning("re-creating image %s from original source "
                                "URL", img["id"])
                    if meta['img_loc'] is not None:
                        self.glance_img_create(
                            img['name'],
                            img['disk_format'] or "qcow2",
                            meta['img_loc']
                        )
                        recreated_image = utl.ext_dict(
                            name=img["name"]
                        )
                        created_images.append((recreated_image, meta))
                    else:
                        raise exception.AbortMigrationError(
                            "image information has no original source URL")
                    continue

                LOG.debug("Creating image '%s' (%s)", img["name"], img['id'])
                # we can face situation when image has no
                # disk_format and container_format properties
                # this situation appears, when image was created
                # with option --copy-from
                # glance-client cannot create image without this
                # properties, we need to create them artificially
                # and then - delete from database

                try:
                    data_proxy = file_like_proxy.FileLikeProxy(
                        img, self.config['migrate']['speed_limit'])

                    created_image = self.create_image(
                        name=img['name'],
                        container_format=(img['container_format'] or "bare"),
                        disk_format=(img['disk_format'] or "qcow2"),
                        is_public=img['is_public'],
                        protected=img['protected'],
                        owner=img['owner'],
                        size=img['size'],
                        properties=img['properties'],
                        data=data_proxy)

                    image_members = img['members'].get(img['id'], {})
                    LOG.debug("new image ID %s", created_image.id)
                    self.update_membership(created_image.id, image_members)
                    created_images.append((created_image, meta))
                except (exception.ImageDownloadError,
                        httplib.IncompleteRead,
                        glance_exceptions.HTTPInternalServerError) as e:
                    LOG.debug(e, exc_info=True)
                    LOG.warning("Unable to reach image's data due to "
                                "Glance HTTPInternalServerError. Skipping "
                                "image: %s (%s)", img['name'], img["id"])
                    obsolete_images_ids_list.append(img["id"])
                    continue

                if not img["container_format"]:
                    delete_container_format.append(created_image.id)
                if not img["disk_format"]:
                    delete_disk_format.append(created_image.id)
            elif img['resource'] is None:
                recreated_image = utl.ext_dict(name=img["name"])
                created_images.append((recreated_image, meta))
            elif not img:
                empty_image_list[image_id_src] = info['images'][image_id_src]

        view.show_progress()
        # Remove obsolete/broken images from info
        for img_id in obsolete_images_ids_list:
            info['images'].pop(img_id)

        if created_images:
            im_name_list = [(im.name, tmp_meta) for (im, tmp_meta) in
                            created_images]
            LOG.debug("images on destination: %s",
                      [im for (im, tmp_meta) in im_name_list])
            new_info = self._convert_images_with_metadata(im_name_list)
        new_info['images'].update(empty_image_list)
        self.delete_fields('disk_format', delete_disk_format)
        self.delete_fields('container_format', delete_container_format)
        LOG.info("Glance images deployment finished.")
        return new_info
Esempio n. 32
0
class ColdEvacuateTestCase(test.TestCase):
    config = utils.ext_dict(
        cloud=utils.ext_dict(
            ssh_user='******',
            ssh_sudo_password='******',
            ssh_host='ssh_host',
            host='host',
        ),
        migrate=utils.ext_dict(
            ssh_chunk_size=1337,
            retry=42,
        ),
    )

    def setUp(self):
        test.TestCase.setUp(self)

        self.server = self._make_server('fake-instance-id', status='ACTive')
        self.servers = {'fake-instance-id': self.server}

        self._services = {}
        self._make_service('nova-compute', 'fake-host-1', 'enabled')
        self._make_service('nova-compute', 'fake-host-2', 'enabled')
        self._make_service('nova-compute', 'fake-host-3', 'disabled')
        self._make_service('nova-compute', 'fake-host-4', 'disabled')
        self._make_service('nova-compute', 'fake-host-5', 'enabled')

        self.compute_api = mock.Mock()
        self.compute_api.servers.get.side_effect = self._servers_get
        self.compute_api.servers.delete.side_effect = self._servers_delete
        self.compute_api.servers.start.side_effect = self._servers_start
        self.compute_api.servers.stop.side_effect = self._servers_stop
        self.compute_api.servers.migrate.side_effect = self._migrate
        self.compute_api.servers.confirm_resize.side_effect = \
            self._confirm_resize

        self.compute_api.services.list.side_effect = self._services_list
        self.compute_api.services.disable.side_effect = self._service_disable
        self.compute_api.services.enable.side_effect = self._service_enable

        cfglib_conf_patcher = mock.patch('cfglib.CONF')
        self.addCleanup(cfglib_conf_patcher.stop)
        self.cfglib_conf = cfglib_conf_patcher.start()
        self.cfglib_conf.evacuation.state_change_timeout = 1
        self.cfglib_conf.evacuation.nova_home_path = '/fake/home'
        self.cfglib_conf.evacuation.nova_user = '******'

        remote_runner_patcher = mock.patch(
            'cloudferrylib.utils.remote_runner.RemoteRunner')
        self.addCleanup(remote_runner_patcher.stop)
        self.remote_runner = remote_runner_patcher.start()

    def _servers_get(self, server_id):
        if not isinstance(server_id, basestring):
            server_id = server_id.id
        if server_id not in self.servers:
            raise nova_exc.NotFound(404)
        return self.servers[server_id]

    def _servers_delete(self, server_id):
        if not isinstance(server_id, basestring):
            server_id = server_id.id
        if server_id not in self.servers:
            raise nova_exc.NotFound(404)
        del self.servers[server_id]

    def _servers_stop(self, server_id):
        self._servers_get(server_id).status = 'SHUTOFF'

    def _servers_start(self, server_id):
        self._servers_get(server_id).status = 'ACTIVE'

    def _migrate(self, server_id):
        server = self._servers_get(server_id)
        server.status = 'VERIFY_RESIZE'
        services = [
            s for s in self._services.values()
            if s.status == 'enabled' and s.binary == 'nova-compute'
            and s.host != getattr(s, cold_evacuate.INSTANCE_HOST_ATTRIBUTE)
        ]
        # concatenate all host names to fail test when there is any choice

        setattr(server, cold_evacuate.INSTANCE_HOST_ATTRIBUTE,
                ','.join(s.host for s in services))

    def _confirm_resize(self, server_id):
        self._servers_get(server_id).status = 'ACTIVE'

    def _make_server(self,
                     instance_id,
                     name='fake-instance',
                     status='active',
                     image='fake-image-id',
                     flavor='fake-flavor-id',
                     availability_zone='fake-az:fake-host',
                     block_device_mapping=None,
                     nics=None):
        if block_device_mapping is None:
            block_device_mapping = {
                '/dev/vdb': 'volume-1',
                '/dev/vdc': 'volume-2'
            }
        _, host = availability_zone.split(':')
        server = mock.Mock()
        server.id = instance_id
        server.name = name
        server.status = status
        server.image = {'id': image}
        server.flavor = {'id': flavor}
        setattr(server, cold_evacuate.INSTANCE_HOST_ATTRIBUTE, host)
        server.block_device_mapping = block_device_mapping
        server.user_id = 'fake-user-id'
        server.nics = nics
        return server

    def _services_list(self, binary=None):
        services = sorted(self._services.values(), key=lambda x: x.host)
        return [s for s in services if s.binary == binary]

    def _make_service(self, binary, host, status):
        service = mock.MagicMock()
        service.binary = binary
        service.host = host
        service.status = status
        self._services[(binary, host)] = service

    def _service_disable(self, host, binary):
        self._services[binary, host].status = 'disabled'

    def _service_enable(self, host, binary):
        self._services[binary, host].status = 'enabled'

    def test_cold_evacuate(self):
        cold_evacuate.cold_evacuate(self.config, self.compute_api, self.server,
                                    'fake-host-5')

        # Check that services are restored after migration
        self.assertEqual(self._services['nova-compute', 'fake-host-1'].status,
                         'enabled')
        self.assertEqual(self._services['nova-compute', 'fake-host-2'].status,
                         'enabled')
        self.assertEqual(self._services['nova-compute', 'fake-host-3'].status,
                         'disabled')
        self.assertEqual(self._services['nova-compute', 'fake-host-4'].status,
                         'disabled')
        self.assertEqual(self._services['nova-compute', 'fake-host-5'].status,
                         'enabled')

        # Check that server migrated to right host
        self.assertEqual(
            getattr(self.server, cold_evacuate.INSTANCE_HOST_ATTRIBUTE),
            'fake-host-5')
Esempio n. 33
0
    def deploy(self, info):
        LOG.info("Glance images deployment started...")
        info = copy.deepcopy(info)
        new_info = {'images': {}}
        migrate_images_list = []
        delete_container_format, delete_disk_format = [], []
        empty_image_list = {}

        # List for obsolete/broken images IDs, that will not be migrated
        obsolete_images_ids_list = []

        for image_id_src, gl_image in info['images'].iteritems():
            if gl_image['image'] and gl_image['image']['resource']:
                dst_img_checksums = {
                    x.checksum: x
                    for x in self.get_image_list()
                }
                dst_img_names = [x.name for x in self.get_image_list()]
                checksum_current = gl_image['image']['checksum']
                name_current = gl_image['image']['name']
                meta = gl_image['meta']
                if checksum_current in dst_img_checksums and (
                        name_current) in dst_img_names:
                    migrate_images_list.append(
                        (dst_img_checksums[checksum_current], meta))
                    continue

                LOG.debug("Updating owner '{owner}' of image '{image}'".format(
                    owner=gl_image["image"]["owner_name"],
                    image=gl_image["image"]["name"]))
                gl_image["image"]["owner"] = \
                    self.identity_client.get_tenant_id_by_name(
                    gl_image["image"]["owner_name"])
                del gl_image["image"]["owner_name"]

                if gl_image["image"]["properties"]:
                    # update snapshot metadata
                    metadata = gl_image["image"]["properties"]
                    if "owner_id" in metadata:
                        # update tenant id
                        LOG.debug("updating snapshot metadata for field "
                                  "'owner_id' for image {image}".format(
                                      image=gl_image["image"]["id"]))
                        metadata["owner_id"] = gl_image["image"]["owner"]
                    if "user_id" in metadata:
                        # update user id by specified name
                        LOG.debug("updating snapshot metadata for field "
                                  "'user_id' for image {image}".format(
                                      image=gl_image["image"]["id"]))
                        metadata["user_id"] = \
                            self.identity_client.keystone_client.users.find(
                                username=metadata["user_name"]).id
                        del metadata["user_name"]
                if gl_image["image"]["checksum"] is None:
                    LOG.warning("re-creating image {} "
                                "from original source URL".format(
                                    gl_image["image"]["id"]))
                    if meta['img_loc'] is not None:
                        self.glance_img_create(
                            gl_image['image']['name'],
                            gl_image['image']['disk_format'] or "qcow2",
                            meta['img_loc'])
                        recreated_image = utl.ext_dict(
                            name=gl_image["image"]["name"])
                        migrate_images_list.append(
                            (recreated_image, gl_image['meta']))
                    else:
                        raise exception.AbortMigrationError(
                            "image information has no original source URL")
                    continue

                LOG.debug("Creating image '{image}' ({image_id})".format(
                    image=gl_image["image"]["name"],
                    image_id=gl_image['image']['id']))
                # we can face situation when image has no
                # disk_format and container_format properties
                # this situation appears, when image was created
                # with option --copy-from
                # glance-client cannot create image without this
                # properties, we need to create them artificially
                # and then - delete from database

                try:
                    migrate_image = self.create_image(
                        name=gl_image['image']['name'],
                        container_format=(gl_image['image']['container_format']
                                          or "bare"),
                        disk_format=(gl_image['image']['disk_format']
                                     or "qcow2"),
                        is_public=gl_image['image']['is_public'],
                        protected=gl_image['image']['protected'],
                        owner=gl_image['image']['owner'],
                        size=gl_image['image']['size'],
                        properties=gl_image['image']['properties'],
                        data=file_like_proxy.FileLikeProxy(
                            gl_image['image'],
                            self.config['migrate']['speed_limit']))
                    LOG.debug("new image ID {}".format(migrate_image.id))
                except exception.ImageDownloadError:
                    LOG.warning(
                        "Unable to reach image's data due to "
                        "Glance HTTPInternalServerError. Skipping "
                        "image: (id = %s)", gl_image["image"]["id"])
                    obsolete_images_ids_list.append(gl_image["image"]["id"])
                    continue

                migrate_images_list.append((migrate_image, meta))
                if not gl_image["image"]["container_format"]:
                    delete_container_format.append(migrate_image.id)
                if not gl_image["image"]["disk_format"]:
                    delete_disk_format.append(migrate_image.id)
            elif gl_image['image']['resource'] is None:
                recreated_image = utl.ext_dict(name=gl_image["image"]["name"])
                migrate_images_list.append((recreated_image, gl_image['meta']))
            elif not gl_image['image']:
                empty_image_list[image_id_src] = gl_image

        # Remove obsolete/broken images from info
        [info['images'].pop(img_id) for img_id in obsolete_images_ids_list]

        if migrate_images_list:
            im_name_list = [(im.name, tmp_meta)
                            for (im, tmp_meta) in migrate_images_list]
            LOG.debug("images on destination: {}".format(
                [im for (im, tmp_meta) in im_name_list]))
            new_info = self.read_info(images_list_meta=im_name_list)
        new_info['images'].update(empty_image_list)
        # on this step we need to create map between source ids and dst ones
        LOG.debug("creating map between source and destination image ids")
        image_ids_map = {}
        dst_img_checksums = {x.checksum: x.id for x in self.get_image_list()}
        for image_id_src, gl_image in info['images'].iteritems():
            cur_image = gl_image["image"]
            image_ids_map[cur_image["id"]] = \
                dst_img_checksums[cur_image["checksum"]]
        LOG.debug("deploying image members")
        for image_id, data in info.get("members", {}).items():
            for tenant_name, can_share in data.items():
                LOG.debug("deploying image member for image {image}"
                          " tenant {tenant}".format(image=image_id,
                                                    tenant=tenant_name))
                self.create_member(image_ids_map[image_id], tenant_name,
                                   can_share)
        self.delete_fields('disk_format', delete_disk_format)
        self.delete_fields('container_format', delete_container_format)
        LOG.info("Glance images deployment finished.")
        return new_info
Esempio n. 34
0
import mock

from keystoneclient.v2_0 import client as keystone_client
from oslotest import mockpatch

from cloudferrylib.os.identity import keystone
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'host': '1.1.1.1'}),
    migrate=utils.ext_dict({'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': '5',
                            'keep_user_passwords': False,
                            'overwrite_user_passwords': False}),
    mail=utils.ext_dict({'server': '-'}))


class KeystoneIdentityTestCase(test.TestCase):
    def setUp(self):
        super(KeystoneIdentityTestCase, self).setUp()
        self.mock_client = mock.MagicMock()
        self.kc_patch = mockpatch.PatchObject(keystone_client, 'Client',
                                              new=self.mock_client)
        self.useFixture(self.kc_patch)
        self.fake_cloud = mock.Mock()
Esempio n. 35
0
    def deploy(self, info, *args, **kwargs):
        LOG.info("Glance images deployment started...")
        info = copy.deepcopy(info)
        created_images = []
        delete_container_format, delete_disk_format = [], []
        empty_image_list = {}

        # List for obsolete/broken images IDs, that will not be migrated
        obsolete_images_ids_list = []
        dst_images = self._dst_images()

        view = GlanceImageProgessMigrationView(info['images'], dst_images)
        view.show_info()
        for image_id_src in info['images']:
            img = info['images'][image_id_src]['image']
            meta = info['images'][image_id_src]['meta']
            if img and img['resource']:
                checksum_current = img['checksum']
                name_current = img['name']
                tenant_name = img['owner_name']
                image_key = (name_current, tenant_name, checksum_current,
                             img['is_public'])

                if image_key in dst_images:
                    existing_image = dst_images[image_key]
                    created_images.append((existing_image, meta))
                    image_members = img['members'].get(img['id'], {})
                    self.update_membership(existing_image.id, image_members)
                    LOG.info(
                        "Image '%s' is already present on destination, "
                        "skipping", img['name'])
                    continue

                view.show_progress()
                view.inc_progress(img['size'])

                LOG.debug("Updating owner '%s' of image '%s'", tenant_name,
                          img["name"])
                img["owner"] = \
                    self.identity_client.get_tenant_id_by_name(tenant_name)

                if img["properties"]:
                    # update snapshot metadata
                    metadata = img["properties"]
                    if "owner_id" in metadata:
                        # update tenant id
                        LOG.debug(
                            "Updating snapshot metadata for field "
                            "'owner_id' for image %s", img["id"])
                        metadata["owner_id"] = img["owner"]
                    if "user_name" in metadata:
                        # update user id by specified name
                        LOG.debug(
                            "Updating snapshot metadata for field "
                            "'user_id' for image %s", img["id"])
                        try:
                            ks_client = self.identity_client.keystone_client
                            metadata["user_id"] = ks_client.users.find(
                                username=metadata["user_name"]).id
                            del metadata["user_name"]
                        except keystone_exceptions.NotFound:
                            LOG.warning("Cannot update user name for image %s",
                                        img['name'])
                if img["checksum"] is None:
                    LOG.warning(
                        "re-creating image %s from original source "
                        "URL", img["id"])
                    if meta['img_loc'] is not None:
                        self.create_image(
                            id=img['id'],
                            name=img['name'],
                            disk_format=img['disk_format'] or "qcow2",
                            location=meta['img_loc'],
                            container_format=img['container_format'] or 'bare',
                        )

                        recreated_image = utl.ext_dict(name=img["name"])
                        created_images.append((recreated_image, meta))
                    else:
                        raise exception.AbortMigrationError(
                            "image information has no original source URL")
                    continue

                LOG.debug("Creating image '%s' (%s)", img["name"], img['id'])
                # we can face situation when image has no
                # disk_format and container_format properties
                # this situation appears, when image was created
                # with option --copy-from
                # glance-client cannot create image without this
                # properties, we need to create them artificially
                # and then - delete from database

                try:
                    file_obj = img['resource'].get_ref_image(img['id'])
                    data_proxy = file_proxy.FileProxy(file_obj,
                                                      name="image %s ('%s')" %
                                                      (img['name'], img['id']),
                                                      size=img['size'])

                    created_image = self.create_image(
                        id=img['id'],
                        name=img['name'],
                        container_format=(img['container_format'] or "bare"),
                        disk_format=(img['disk_format'] or "qcow2"),
                        is_public=img['is_public'],
                        protected=img['protected'],
                        owner=img['owner'],
                        size=img['size'],
                        properties=img['properties'],
                        data=data_proxy)

                    image_members = img['members'].get(img['id'], {})
                    LOG.debug("new image ID %s", created_image.id)
                    self.update_membership(created_image.id, image_members)
                    created_images.append((created_image, meta))
                except (exception.ImageDownloadError, httplib.IncompleteRead,
                        glance_exceptions.HTTPInternalServerError) as e:
                    LOG.debug(e, exc_info=True)
                    LOG.warning(
                        "Unable to reach image's data due to "
                        "Glance HTTPInternalServerError. Skipping "
                        "image: %s (%s)", img['name'], img["id"])
                    obsolete_images_ids_list.append(img["id"])
                    continue

                if not img["container_format"]:
                    delete_container_format.append(created_image.id)
                if not img["disk_format"]:
                    delete_disk_format.append(created_image.id)
            elif img['resource'] is None:
                recreated_image = utl.ext_dict(name=img["name"])
                created_images.append((recreated_image, meta))
            elif not img:
                empty_image_list[image_id_src] = info['images'][image_id_src]

        view.show_progress()
        if obsolete_images_ids_list:
            LOG.warning('List of broken images: %s', obsolete_images_ids_list)
            # Remove obsolete/broken images from info
            for img_id in obsolete_images_ids_list:
                info['images'].pop(img_id)

        return self._new_info(created_images, empty_image_list,
                              delete_disk_format, delete_container_format)
Esempio n. 36
0
from oslotest import mockpatch

import cfglib
from cloudferrylib.os.identity import keystone
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'region': None,
                          'service_tenant': 'service',
                          'cacert': '',
                          'insecure': False}),
    migrate=utils.ext_dict({'retry': '7',
                            'time_wait': 5,
                            'keep_user_passwords': False,
                            'overwrite_user_passwords': False,
                            'migrate_users': True,
                            'optimize_user_role_fetch': False}),
    mail=utils.ext_dict({'server': '-'}))


@mock.patch("cloudferrylib.base.clients.os_cli_cmd", mock.MagicMock())
class KeystoneIdentityTestCase(test.TestCase):
    def setUp(self):
        super(KeystoneIdentityTestCase, self).setUp()
        self.mock_client = mock.MagicMock()
        self.kc_patch = mockpatch.PatchObject(keystone_client, 'Client',
Esempio n. 37
0
from glanceclient.v1 import client as glance_client
from oslotest import mockpatch

from cloudferrylib.os.image.glance_image import GlanceImage
from cloudferrylib.utils import utils
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'region': None,
    'host': '1.1.1.1',
    'ssh_host': '1.1.1.10',
    'ssh_user': '******',
    'cacert': '',
    'insecure': False
}),
                             migrate=utils.ext_dict({
                                 'retry': '7',
                                 'time_wait': 5
                             }))


class FakeUser(object):
    def __init__(self):
        self.name = 'fake_user_name'


class GlanceImageTestCase(test.TestCase):
Esempio n. 38
0
from keystoneclient.v2_0 import client as keystone_client
from oslotest import mockpatch

from cloudferrylib.os.identity import keystone
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'service_tenant': 'services'}),
    migrate=utils.ext_dict({'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': 5,
                            'keep_user_passwords': False,
                            'overwrite_user_passwords': False,
                            'migrate_users': True}),
    mail=utils.ext_dict({'server': '-'}))


class KeystoneIdentityTestCase(test.TestCase):
    def setUp(self):
        super(KeystoneIdentityTestCase, self).setUp()
        self.mock_client = mock.MagicMock()
        self.kc_patch = mockpatch.PatchObject(keystone_client, 'Client',
                                              new=self.mock_client)
        self.useFixture(self.kc_patch)
Esempio n. 39
0

import mock

from glanceclient.v1 import client as glance_client
from oslotest import mockpatch

from cloudferrylib.os.image.glance_image import GlanceImage
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({'user': '******',
                                                   'password': '******',
                                                   'tenant': 'fake_tenant',
                                                   'host': '1.1.1.1',
                                                   }),
                             migrate=utils.ext_dict({'speed_limit': '10MB',
                                                     'retry': '7',
                                                     'time_wait': '5'}))


class GlanceImageTestCase(test.TestCase):
    def setUp(self):
        super(GlanceImageTestCase, self).setUp()

        self.glance_mock_client = mock.MagicMock()
        self.glance_mock_client().images.data()._resp = 'fake_resp_1'

        self.glance_client_patch = mockpatch.PatchObject(
            glance_client,
            'Client',
Esempio n. 40
0
from oslotest import mockpatch
import cfglib

from cloudferrylib.os.compute import nova_compute
from cloudferrylib.utils import timeout_exception
from cloudferrylib.utils import utils

from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'region': None,
                          'auth_url': 'http://1.1.1.1:35357/v2.0/'}),
    mysql=utils.ext_dict({'host': '1.1.1.1'}),
    migrate=utils.ext_dict({'migrate_quotas': True,
                            'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': 5}))


class NovaComputeTestCase(test.TestCase):
    def setUp(self):
        super(NovaComputeTestCase, self).setUp()

        self.mock_client = mock.MagicMock()
        self.nc_patch = mockpatch.PatchObject(nova_client, 'Client',
                                              new=self.mock_client)
        self.useFixture(self.nc_patch)
Esempio n. 41
0
from neutronclient.v2_0 import client as neutron_client
from oslotest import mockpatch

from cloudferrylib.base import exception
from cloudferrylib.os.network import neutron
from cloudferrylib.utils import utils
from tests import test

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({
    'user': '******',
    'password': '******',
    'tenant': 'fake_tenant',
    'auth_url': 'http://1.1.1.1:35357/v2.0/',
    'region': None,
    'service_tenant': 'services',
    'cacert': '',
    'insecure': False
}),
                             migrate=utils.ext_dict({
                                 'ext_net_map': 'fake_ext_net_map.yaml',
                                 'retry': '7',
                                 'time_wait': 5
                             }),
                             network=utils.ext_dict({'get_all_quota': True}))


class NeutronTestCase(test.TestCase):
    def setUp(self):
        super(NeutronTestCase, self).setUp()

        self.neutron_mock_client = mock.MagicMock()
Esempio n. 42
0
from oslotest import mockpatch

from cloudferrylib.os.compute import nova_compute
from cloudferrylib.utils import timeout_exception
from cloudferrylib.utils import utils

from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'region': None,
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'cacert': '',
                          'insecure': False}),
    mysql=utils.ext_dict({'host': '1.1.1.1'}),
    migrate=utils.ext_dict({'migrate_quotas': True,
                            'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': 5}))


class NovaComputeTestCase(test.TestCase):
    def setUp(self):
        super(NovaComputeTestCase, self).setUp()

        self.mock_client = mock.MagicMock()
        self.nc_patch = mockpatch.PatchObject(nova_client, 'Client',
                                              new=self.mock_client)
Esempio n. 43
0
import mock

from neutronclient.v2_0 import client as neutron_client
from oslotest import mockpatch

from cloudferrylib.os.network import neutron
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict({'user': '******',
                          'password': '******',
                          'tenant': 'fake_tenant',
                          'auth_url': 'http://1.1.1.1:35357/v2.0/',
                          'region': None,
                          'service_tenant': 'services'}),
    migrate=utils.ext_dict({'ext_net_map': 'fake_ext_net_map.yaml',
                            'speed_limit': '10MB',
                            'retry': '7',
                            'time_wait': 5}))


class NeutronTestCase(test.TestCase):

    def setUp(self):
        super(NeutronTestCase, self).setUp()

        self.neutron_mock_client = mock.MagicMock()

        self.neutron_client_patch = \
Esempio n. 44
0
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import mock

from cloudferrylib.os.storage import cinder_storage
from tests import test
from oslotest import mockpatch
from cinderclient.v1 import client as cinder_client
from cloudferrylib.utils import utils

FAKE_CONFIG = utils.ext_dict(cloud=utils.ext_dict({'user': '******',
                                                   'password': '******',
                                                   'tenant': 'fake_tenant',
                                                   'host': '1.1.1.1'}))


class CinderStorageTestCase(test.TestCase):
    def setUp(self):
        super(CinderStorageTestCase, self).setUp()
        self.mock_client = mock.Mock()
        self.cs_patch = mockpatch.PatchObject(cinder_client, 'Client',
                                              new=self.mock_client)
        self.useFixture(self.cs_patch)

        self.identity_mock = mock.Mock()

        self.fake_cloud = mock.Mock()
        self.fake_cloud.mysql_connector = mock.Mock()
Esempio n. 45
0
from glanceclient.v1 import client as glance_client
from oslotest import mockpatch

from cloudferrylib.os.image.glance_image import GlanceImage
from cloudferrylib.utils import utils
from tests import test


FAKE_CONFIG = utils.ext_dict(
    cloud=utils.ext_dict(
        {
            "user": "******",
            "password": "******",
            "tenant": "fake_tenant",
            "region": None,
            "host": "1.1.1.1",
            "ssh_user": "******",
        }
    ),
    migrate=utils.ext_dict({"speed_limit": "10MB", "retry": "7", "time_wait": 5}),
)


class FakeUser(object):
    def __init__(self):
        self.name = "fake_user_name"


class GlanceImageTestCase(test.TestCase):
    def setUp(self):