def main(profile, zwps_to_cwps, add_affinity_group, destination_dc, is_project_vm,
skip_within_cluster, dry_run, vm, cluster):
"""Live migrate VM to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Live Migrate VM'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
# Work around migration issue: first in the same pod to limit possible hiccup
vm_instance = co.get_vm(name=vm, is_project_vm=is_project_vm)
if not vm_instance:
logging.error(f"Cannot migrate, VM '{vm}' not found!")
sys.exit(1)
if not vm_instance['state'] == 'Running':
logging.error(f"Cannot migrate, VM has has state: '{vm_instance['state']}'")
sys.exit(1)
source_host = co.get_host(id=vm_instance['hostid'])
source_cluster = co.get_cluster(id=source_host['clusterid'])
if not skip_within_cluster:
if not vm_instance.migrate_within_cluster(vm=vm_instance, source_cluster=source_cluster,
source_host=source_host, instancename=vm_instance):
logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
sys.exit(1)
if not live_migrate(co, cs, cluster, vm, destination_dc, add_affinity_group, is_project_vm, zwps_to_cwps,
log_to_slack, dry_run):
logging.info(f"VM Migration failed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
sys.exit(1)
logging.info(f"VM Migration completed at {datetime.now().strftime('%d-%m-%Y %H:%M:%S')}\n")
def main(profile, is_project_vm, dry_run, vm, cluster, destination_dc, destination_so):
"""Offline migrate VM to CLUSTER"""
click_log.basic_config()
log_to_slack = True
logging.task = 'Offline Migrate VM'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
cs = CosmicSQL(server=profile, dry_run=dry_run)
target_cluster = co.get_cluster(name=cluster)
if not target_cluster:
sys.exit(1)
vm = co.get_vm(name=vm, is_project_vm=is_project_vm)
if not vm:
sys.exit(1)
if destination_dc and destination_dc not in DATACENTERS:
logging.error(f"Unknown datacenter '{destination_dc}', should be one of {str(DATACENTERS)}")
sys.exit(1)
logging.instance_name = vm['instancename']
logging.slack_value = vm['domain']
logging.vm_name = vm['name']
logging.zone_name = vm['zonename']
target_storage_pool = None
try:
# Get CLUSTER scoped volume (no NVMe or ZONE-wide)
while target_storage_pool is None or target_storage_pool['scope'] != 'CLUSTER':
target_storage_pool = choice(target_cluster.get_storage_pools())
except IndexError:
logging.error(f"No storage pools found for cluster '{target_cluster['name']}")
sys.exit(1)
if vm['state'] == 'Running':
need_to_stop = True
auto_start_vm = True
else:
need_to_stop = False
auto_start_vm = False
if destination_dc:
for datacenter in DATACENTERS:
if datacenter == destination_dc:
continue
if datacenter in vm['serviceofferingname']:
new_offering = vm['serviceofferingname'].replace(datacenter, destination_dc)
logging.info(
f"Replacing '{vm['serviceofferingname']}' with '{new_offering}'")
cs.update_service_offering_of_vm(vm['instancename'], new_offering)
vm = co.get_vm(name=vm['instancename'], is_project_vm=is_project_vm)
break
if destination_so:
logging.info(
f"Replacing '{vm['serviceofferingname']}' with '{destination_so}'")
cs.update_service_offering_of_vm(vm['instancename'], destination_so)
vm = co.get_vm(name=vm['instancename'], is_project_vm=is_project_vm)
vm_service_offering = co.get_service_offering(id=vm['serviceofferingid'])
if vm_service_offering:
storage_tags = vm_service_offering['tags'] if 'tags' in vm_service_offering else ''
if not storage_tags:
logging.warning('VM service offering has no storage tags')
else:
if storage_tags not in target_storage_pool['tags']:
logging.error(
f"Can't migrate '{vm['name']}', storage tags on target cluster ({target_storage_pool['tags']}) to not contain the tags on the VM's service offering ({storage_tags})'")
sys.exit(1)
if need_to_stop:
if not vm.stop():
sys.exit(1)
volumes = vm.get_volumes()
for volume in volumes:
if volume['storage'] == target_storage_pool['name']:
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) already on cluster '{target_cluster['name']}', skipping...")
continue
source_storage_pool = co.get_storage_pool(name=volume['storage'])
if not source_storage_pool:
sys.exit(1)
if source_storage_pool['scope'] == 'ZONE':
logging.warning(f"Scope of volume '{volume['name']}' ({volume['id']}) is ZONE, skipping...")
continue
if not volume.migrate(target_storage_pool):
sys.exit(1)
with click_spinner.spinner():
while True:
volume.refresh()
if volume['state'] == 'Ready':
break
logging.warning(
f"Volume '{volume['name']}' ({volume['id']}) is in '{volume['state']}' state instead of 'Ready', sleeping...")
time.sleep(60)
if auto_start_vm:
destination_host = target_cluster.find_migration_host(vm)
if not destination_host:
sys.exit(1)
if not vm.start(destination_host):
sys.exit(1)
else:
logging.info(f"Not starting VM '{vm['name']}' as it was not running", log_to_slack)
def main(dry_run, zwps_cluster, destination_cluster, virtual_machines,
force_end_hour):
"""Empty ZWPS by migrating VMs and/or it's volumes to the destination cluster."""
click_log.basic_config()
if force_end_hour:
try:
force_end_hour = int(force_end_hour)
except ValueError as e:
logging.error(
f"Specified time:'{force_end_hour}' is not a valid integer due to: '{e}'"
)
sys.exit(1)
if force_end_hour >= 24:
logging.error(f"Specified time:'{force_end_hour}' should be < 24")
sys.exit(1)
profile = 'nl2'
log_to_slack = True
logging.task = 'Live Migrate VM Volumes'
logging.slack_title = 'Domain'
if dry_run:
log_to_slack = False
logging.warning('Running in dry-run mode, will only show changes')
co = CosmicOps(profile=profile, dry_run=dry_run, log_to_slack=log_to_slack)
if not dry_run:
cs = CosmicSQL(server=profile, dry_run=dry_run)
else:
cs = None
zwps_storage_pools = []
for storage_pool in co.get_all_storage_pools():
if zwps_cluster.upper() in storage_pool['name']:
zwps_storage_pools.append(storage_pool)
logging.info('ZWPS storage pools found:')
for zwps_storage_pool in zwps_storage_pools:
logging.info(f" - '{zwps_storage_pool['name']}'")
target_cluster = co.get_cluster(name=destination_cluster)
if not target_cluster:
logging.error(
f"Destination cluster not found:'{target_cluster['name']}'!")
sys.exit(1)
try:
destination_storage_pools = target_cluster.get_storage_pools(
scope='CLUSTER')
except IndexError:
logging.error(
f"No storage pools found for cluster '{target_cluster['name']}'")
sys.exit(1)
logging.info('Destination storage pools found:')
for target_storage_pool in destination_storage_pools:
logging.info(f" - '{target_storage_pool['name']}'")
target_storage_pool = random.choice(destination_storage_pools)
volumes = []
for zwps_storage_pool in zwps_storage_pools:
vols = co.get_all_volumes(list_all=True,
storageid=zwps_storage_pool['id'])
if vols:
volumes += vols
vm_ids = []
logging.info('Volumes found:')
for volume in volumes:
for virtual_machine in virtual_machines:
if re.search(virtual_machine, volume['vmname'], re.IGNORECASE):
logging.info(
f" - '{volume['name']}' on VM '{volume['vmname']}'")
if volume['virtualmachineid'] not in vm_ids:
vm_ids.append(volume['virtualmachineid'])
vms = []
for vm_id in vm_ids:
vm = co.get_vm(id=vm_id)
if vm['affinitygroup']:
for affinitygroup in vm['affinitygroup']:
if 'DedicatedGrp' in affinitygroup['name']:
logging.warning(
f"Skipping VM '{vm['name']}' because of 'DedicatedGrp' affinity group"
)
continue
vms.append(vm)
logging.info('Virtualmachines found:')
for vm in vms:
logging.info(f" - '{vm['name']}'")
logging.info(
f"Starting live migration of volumes and/or virtualmachines from the ZWPS storage pools to storage pool '{target_cluster['name']}'"
)
for vm in vms:
""" Can we start a new migration? """
if force_end_hour:
now = datetime.datetime.now(pytz.timezone('CET'))
if now.hour >= force_end_hour:
logging.info(
f"Stopping migration batch. We are not starting new migrations after '{force_end_hour}':00",
log_to_slack=log_to_slack)
sys.exit(0)
source_host = co.get_host(id=vm['hostid'])
source_cluster = co.get_cluster(zone='nl2',
id=source_host['clusterid'])
if source_cluster['name'] == target_cluster['name']:
""" VM is already on the destination cluster, so we only need to migrate the volumes to this storage pool """
logging.info(
f"Starting live migration of volumes of VM '{vm['name']}' to storage pool '{target_storage_pool['name']}' ({target_storage_pool['id']})",
log_to_slack=log_to_slack)
live_migrate_volumes(target_storage_pool['name'], co, cs, dry_run,
False, log_to_slack, 0, vm['name'], True)
else:
""" VM needs to be migrated live to the destination cluster, including volumes """
live_migrate(co=co,
cs=cs,
cluster=target_cluster['name'],
vm_name=vm['name'],
destination_dc=None,
add_affinity_group=None,
is_project_vm=None,
zwps_to_cwps=True,
log_to_slack=log_to_slack,
dry_run=dry_run)
class TestCosmicOps(TestCase):
def setUp(self):
cs_patcher = patch('cosmicops.ops.CloudStack')
self.mock_cs = cs_patcher.start()
self.addCleanup(cs_patcher.stop)
self.cs_instance = self.mock_cs.return_value
slack_patcher = patch('cosmicops.log.Slack')
self.mock_slack = slack_patcher.start()
self.addCleanup(slack_patcher.stop)
sleep_patcher = patch('time.sleep', return_value=None)
self.mock_sleep = sleep_patcher.start()
self.addCleanup(sleep_patcher.stop)
self.co = CosmicOps(endpoint='https://localhost',
key='key',
secret='secret')
@patch('cosmicops.ops._load_cloud_monkey_profile')
def test_init_with_profile(self, mock_load):
mock_load.return_value = ('profile_endpoint', 'profile_key',
'profile_secret')
CosmicOps(profile='config')
self.mock_cs.assert_called_with('profile_endpoint', 'profile_key',
'profile_secret', 60)
@tempdir()
def test_load_cloud_monkey_profile(self, tmp):
config = (b"[testprofile]\n"
b"url = http://localhost:8000/client/api\n"
b"apikey = test_api_key\n"
b"secretkey = test_secret_key\n")
tmp.makedir('.cloudmonkey')
tmp.write('.cloudmonkey/config', config)
with patch('pathlib.Path.home') as path_home_mock:
path_home_mock.return_value = Path(tmp.path)
(endpoint, key, secret) = _load_cloud_monkey_profile('testprofile')
self.assertEqual('http://localhost:8000/client/api', endpoint)
self.assertEqual('test_api_key', key)
self.assertEqual('test_secret_key', secret)
@tempdir()
def test_load_cloud_monkey_profile_with_default(self, tmp):
config = (b"[core]\n"
b"profile = profile2\n\n"
b"[profile1]\n"
b"url = http://localhost:8000/client/api/1\n"
b"apikey = test_api_key_1\n"
b"secretkey = test_secret_key_1\n\n"
b"[profile2]\n"
b"url = http://localhost:8000/client/api/2\n"
b"apikey = test_api_key_2\n"
b"secretkey = test_secret_key_2\n")
tmp.makedir('.cloudmonkey')
tmp.write('.cloudmonkey/config', config)
with patch('pathlib.Path.home') as path_home_mock:
path_home_mock.return_value = Path(tmp.path)
(endpoint, key, secret) = _load_cloud_monkey_profile('config')
self.assertEqual('http://localhost:8000/client/api/2', endpoint)
self.assertEqual('test_api_key_2', key)
self.assertEqual('test_secret_key_2', secret)
def test_cs_get_single_result(self):
self.cs_instance.listFunction.return_value = [{
'id': 'id_field',
'name': 'name_field'
}]
result = self.co._cs_get_single_result('listFunction',
{'name': 'name_field'},
CosmicObject, 'type')
self.cs_instance.listFunction.assert_called_with(fetch_list=True,
name='name_field')
self.assertIsInstance(result, CosmicObject)
self.assertDictEqual({
'id': 'id_field',
'name': 'name_field'
}, result._data)
def test_get_get_single_result_failure(self):
self.cs_instance.listFunction.return_value = []
self.assertIsNone(
self.co._cs_get_single_result('listFunction', {}, CosmicObject,
'type'))
self.cs_instance.listFunction.return_value = [{}, {}]
self.assertIsNone(
self.co._cs_get_single_result('listFunction', {}, CosmicObject,
'type'))
def test_cs_get_all_results(self):
self.cs_instance.listFunction.return_value = [{
'id': 'id1',
'name': 'name1'
}, {
'id': 'id2',
'name': 'name2'
}]
result = self.co._cs_get_all_results('listFunction', {}, CosmicObject,
'type')
self.cs_instance.listFunction.assert_called_with(fetch_list=True)
for i, item in enumerate(result):
self.assertIsInstance(item, CosmicObject)
self.assertDictEqual({
'id': f'id{i + 1}',
'name': f'name{i + 1}'
}, item._data)
def test_get_vm(self):
self.co._cs_get_single_result = Mock()
self.co.get_vm(name='vm1')
self.assertDictEqual({
'name': 'vm1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_vm(name='i-VM-1')
self.assertDictEqual({
'keyword': 'i-VM-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_vm(name='project_vm1', is_project_vm=True)
self.assertDictEqual(
{
'name': 'project_vm1',
'projectid': '-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_project_vm(self):
self.co._cs_get_single_result = Mock()
self.co.get_project_vm(name='project_vm1')
self.assertDictEqual(
{
'name': 'project_vm1',
'projectid': '-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_router(self):
self.co._cs_get_single_result = Mock()
self.co.get_router(name='router1')
self.assertDictEqual({
'name': 'router1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_router(name='project_router1', is_project_router=True)
self.assertDictEqual(
{
'name': 'project_router1',
'projectid': '-1',
'listall': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_cluster(self):
self.co._cs_get_single_result = Mock()
self.co.get_zone = Mock(
return_value=CosmicZone(Mock(), {
'id': 'z1',
'name': 'zone1'
}))
self.co.get_cluster(name='cluster1')
self.assertDictEqual({'name': 'cluster1'},
self.co._cs_get_single_result.call_args[0][1])
self.co.get_cluster(name='cluster1', zone='zone1')
self.assertDictEqual({
'name': 'cluster1',
'zoneid': 'z1'
}, self.co._cs_get_single_result.call_args[0][1])
self.co.get_zone.return_value = None
self.assertIsNone(self.co.get_cluster(name='cluster1', zone='zone1'))
def test_get_all_clusters(self):
self.co._cs_get_all_results = Mock()
self.co.get_all_clusters()
self.assertDictEqual({}, self.co._cs_get_all_results.call_args[0][1])
self.co.get_all_clusters(zone=CosmicZone(Mock(), {'id': 'z1'}))
self.assertDictEqual({'zoneid': 'z1'},
self.co._cs_get_all_results.call_args[0][1])
self.co.get_all_clusters(pod=CosmicPod(Mock(), {'id': 'p1'}))
self.assertDictEqual({'podid': 'p1'},
self.co._cs_get_all_results.call_args[0][1])
def test_get_service_offering(self):
self.co._cs_get_single_result = Mock()
self.co.get_service_offering(name='so1')
self.assertDictEqual({'name': 'so1'},
self.co._cs_get_single_result.call_args[0][1])
self.co.get_service_offering(name='so1', system=True)
self.assertDictEqual({
'name': 'so1',
'issystem': True
}, self.co._cs_get_single_result.call_args[0][1])
def test_get_all_vms(self):
self.co._cs_get_all_results = Mock()
self.co.get_all_vms()
self.assertDictEqual({'listall': True},
self.co._cs_get_all_results.call_args[0][1])
def test_get_all_project_vms(self):
self.co._cs_get_all_results = Mock()
self.co.get_all_project_vms()
self.assertDictEqual({
'projectid': '-1',
'listall': True
}, self.co._cs_get_all_results.call_args[0][1])
def test_wait_for_job(self):
self.cs_instance.queryAsyncJobResult.return_value = {'jobstatus': '1'}
self.assertTrue(self.co.wait_for_job('job'))
def test_wait_for_job_cloud_stack_exception(self):
self.cs_instance.queryAsyncJobResult.side_effect = CloudStackException(
response=Mock())
self.assertRaises(CloudStackException, self.co.wait_for_job, 'job')
def test_wait_for_job_connection_error(self):
self.cs_instance.queryAsyncJobResult.side_effect = ConnectionError
self.assertRaises(ConnectionError, self.co.wait_for_job, 'job')
def test_wait_for_job_retries(self):
self.cs_instance.queryAsyncJobResult.side_effect = [
CloudStackException('multiple JSON fields named jobstatus',
response=Mock()),
ConnectionError('Connection aborted'),
ConnectionError('Connection aborted')
]
self.assertFalse(self.co.wait_for_job('job', 3))
def test_wait_for_job_failure(self):
self.cs_instance.queryAsyncJobResult.return_value = {'jobstatus': '2'}
self.assertFalse(self.co.wait_for_job('job'))
